diff --git a/3rdParty/iresearch/core/error/error.cpp b/3rdParty/iresearch/core/error/error.cpp index 41e6fd070b..adae32b9fd 100644 --- a/3rdParty/iresearch/core/error/error.cpp +++ b/3rdParty/iresearch/core/error/error.cpp @@ -74,52 +74,6 @@ const char* eof_error::what() const NOEXCEPT { return "Read past EOF."; } -// ---------------------------------------------------------------------------- -// detailed_io_error -// ---------------------------------------------------------------------------- - -detailed_io_error::detailed_io_error( - const irs::string_ref& error /*= irs::string_ref::NIL*/ -) { - if (!error.empty()) { - error_.append(error.c_str(), error.size()); - } -} - -detailed_io_error::detailed_io_error(std::string&& error) - : error_(std::move(error)) { -} - -detailed_io_error::detailed_io_error(const char* error) - : detailed_io_error(irs::string_ref(error)) { -} - -detailed_io_error& detailed_io_error::operator<<(const irs::string_ref& error) { - if (!error.empty()) { - error_.append(error.c_str(), error.size()); - } - - return *this; -} - -detailed_io_error& detailed_io_error::operator<<(std::string&& error) { - error.append(std::move(error)); - - return *this; -} - -detailed_io_error& detailed_io_error::operator<<(const char* error) { - return (*this) << irs::string_ref(error); -} - -ErrorCode detailed_io_error::code() const NOEXCEPT { - return CODE; -} - -const char* detailed_io_error::what() const NOEXCEPT { - return error_.c_str(); -} - // ---------------------------------------------------------------------------- // lock_obtain_failed // ---------------------------------------------------------------------------- @@ -178,18 +132,6 @@ const char* index_not_found::what() const NOEXCEPT { return "No segments* file found."; } -// ---------------------------------------------------------------------------- -// index_error -// ---------------------------------------------------------------------------- - -ErrorCode index_error::code() const NOEXCEPT{ - return CODE; -} - -const char* index_error::what() const NOEXCEPT { - return "Index error."; -} - // ---------------------------------------------------------------------------- // not_impl_error // ---------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/core/error/error.hpp b/3rdParty/iresearch/core/error/error.hpp index e671ee0176..8678767765 100644 --- a/3rdParty/iresearch/core/error/error.hpp +++ b/3rdParty/iresearch/core/error/error.hpp @@ -24,10 +24,10 @@ #ifndef IRESEARCH_ERROR_H #define IRESEARCH_ERROR_H -#include "utils/string.hpp" - #include +#include "utils/string.hpp" + MSVC_ONLY(class IRESEARCH_API std::exception); NS_ROOT @@ -57,6 +57,34 @@ struct IRESEARCH_API error_base: std::exception { virtual const char* what() const NOEXCEPT override; }; +// ----------------------------------------------------------------------------- +// detailed_error_base +// ----------------------------------------------------------------------------- +class IRESEARCH_API detailed_error_base: public error_base { + public: + explicit detailed_error_base(std::string&& error) NOEXCEPT + : error_(std::move(error)) { + } + + detailed_error_base(const detailed_error_base& other) + : error_(other.error_) { + } + + detailed_error_base(detailed_error_base&& other) NOEXCEPT + : error_(std::move(other.error_)) { + } + + detailed_error_base& operator=(const detailed_error_base& other) = delete; + detailed_error_base& operator=(detailed_error_base&& other) NOEXCEPT = delete; + + virtual const char* what() const NOEXCEPT final { return error_.c_str(); } + + private: + IRESEARCH_API_PRIVATE_VARIABLES_BEGIN + std::string error_; + IRESEARCH_API_PRIVATE_VARIABLES_END +}; + // ---------------------------------------------------------------------------- // not_supported // ---------------------------------------------------------------------------- @@ -87,21 +115,14 @@ struct IRESEARCH_API eof_error: io_error { // ---------------------------------------------------------------------------- // detailed_io_error // ---------------------------------------------------------------------------- -class IRESEARCH_API detailed_io_error: public io_error { +struct IRESEARCH_API detailed_io_error: public detailed_error_base { public: + explicit detailed_io_error(std::string&& error) + : detailed_error_base(std::move(error)) { + } + DECLARE_ERROR_CODE(io_error); - explicit detailed_io_error(const irs::string_ref& error = irs::string_ref::NIL); - explicit detailed_io_error(std::string&& error); - explicit detailed_io_error(const char* error); - detailed_io_error& operator<<(const irs::string_ref& error); - detailed_io_error& operator<<(std::string&& error); - detailed_io_error& operator<<(const char* error); - virtual iresearch::ErrorCode code() const NOEXCEPT override; - virtual const char* what() const NOEXCEPT override; - private: - IRESEARCH_API_PRIVATE_VARIABLES_BEGIN - std::string error_; - IRESEARCH_API_PRIVATE_VARIABLES_END + virtual ErrorCode code() const NOEXCEPT override { return CODE; } }; // ---------------------------------------------------------------------------- @@ -146,10 +167,13 @@ struct IRESEARCH_API index_not_found: error_base { // ---------------------------------------------------------------------------- // index_error // ---------------------------------------------------------------------------- -struct IRESEARCH_API index_error: error_base { - DECLARE_ERROR_CODE( index_error ); - virtual ErrorCode code() const NOEXCEPT override; - virtual const char* what() const NOEXCEPT override; +struct IRESEARCH_API index_error: detailed_error_base { + explicit index_error(std::string&& error) + : detailed_error_base(std::move(error)) { + } + + DECLARE_ERROR_CODE(index_error); + virtual ErrorCode code() const NOEXCEPT override { return CODE; } }; // ---------------------------------------------------------------------------- @@ -181,4 +205,4 @@ struct IRESEARCH_API illegal_state: error_base { NS_END -#endif \ No newline at end of file +#endif diff --git a/3rdParty/iresearch/core/formats/format_utils.cpp b/3rdParty/iresearch/core/formats/format_utils.cpp index a2256f9418..64ab887d1f 100644 --- a/3rdParty/iresearch/core/formats/format_utils.cpp +++ b/3rdParty/iresearch/core/formats/format_utils.cpp @@ -28,25 +28,33 @@ #include "formats/formats.hpp" +#include + NS_ROOT void validate_footer(iresearch::index_input& in) { const int64_t remain = in.length() - in.file_pointer(); + if (remain != format_utils::FOOTER_LEN) { - // invalid position - throw iresearch::index_error(); + throw index_error( + std::string("while validating footer, error: invalid position '") + std::to_string(remain) + "'" + ); } const int32_t magic = in.read_int(); + if (magic != format_utils::FOOTER_MAGIC) { - // invalid magic number - throw iresearch::index_error(); + throw index_error( + std::string("while validating footer, error: invalid magic number '") + std::to_string(magic) + "'" + ); } const int32_t alg_id = in.read_int(); + if (alg_id != 0) { - // invalid algorithm - throw iresearch::index_error(); + throw index_error( + std::string("while validating footer, error: invalid algorithm '") + std::to_string(alg_id) + "'" + ); } } @@ -69,25 +77,34 @@ int32_t check_header( const string_ref& req_format, int32_t min_ver, int32_t max_ver) { const int32_t magic = in.read_int(); + if (FORMAT_MAGIC != magic) { - // index format - throw index_error(); + throw irs::index_error( + std::string("while checking header, error: invalid magic '") + std::to_string(magic) + "'" + ); } const auto format = read_string(in); + if (compare(req_format, format) != 0) { - // invalid format - throw index_error(); + std::stringstream ss; + + ss << "while checking header, error: format mismatch '" << format << "' != '" << req_format << "'"; + + throw irs::index_error(ss.str()); } const int32_t ver = in.read_int(); + if (ver < min_ver || ver > max_ver) { - // invalid version - throw index_error(); + throw irs::index_error( + std::string("while checking header, error: invalid version '") + std::to_string(ver) + "'" + ); } return ver; } NS_END + NS_END diff --git a/3rdParty/iresearch/core/formats/format_utils.hpp b/3rdParty/iresearch/core/formats/format_utils.hpp index 5f8028bc6a..b90105e94d 100644 --- a/3rdParty/iresearch/core/formats/format_utils.hpp +++ b/3rdParty/iresearch/core/formats/format_utils.hpp @@ -81,8 +81,9 @@ inline int64_t check_footer(index_input& in, int64_t checksum) { validate_footer(in); if (checksum != in.read_long()) { - // invalid checksum - throw index_error(); + throw index_error( + std::string("while checking footer, error: invalid checksum '") + std::to_string(checksum) + "'" + ); } return checksum; diff --git a/3rdParty/iresearch/core/formats/formats_10.cpp b/3rdParty/iresearch/core/formats/formats_10.cpp index a94ca42007..1cfda93bfa 100644 --- a/3rdParty/iresearch/core/formats/formats_10.cpp +++ b/3rdParty/iresearch/core/formats/formats_10.cpp @@ -217,7 +217,7 @@ inline void prepare_output( if (!out) { std::stringstream ss; - ss << "Failed to create file, path: " << str; + ss << "failed to create file, path: " << str; throw detailed_io_error(ss.str()); } @@ -241,7 +241,7 @@ inline void prepare_input( if (!in) { std::stringstream ss; - ss << "Failed to open file, path: " << str; + ss << "failed to open file, path: " << str; throw detailed_io_error(ss.str()); } @@ -652,8 +652,9 @@ void postings_writer::begin_doc(doc_id_t id, const frequency* freq) { } if (id < doc.last) { - // docs out of order - throw index_error(); + throw index_error( + std::string("while beginning doc in postings_writer, error: docs out of order '") + std::to_string(id) + "' < '" + std::to_string(doc.last) + "'" + ); } doc.doc(id - doc.last); @@ -999,7 +1000,7 @@ class doc_iterator : public iresearch::doc_iterator { if (!doc_in_) { IR_FRMT_FATAL("Failed to reopen document input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen document input"); + throw detailed_io_error("failed to reopen document input"); } } @@ -1354,7 +1355,7 @@ class pos_iterator: public position { if (!pos_in_) { IR_FRMT_FATAL("Failed to reopen positions input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen positions input"); + throw detailed_io_error("failed to reopen positions input"); } pos_in_->seek(state.term_state->pos_start); @@ -1467,7 +1468,7 @@ class offs_pay_iterator final: public pos_iterator { if (!pay_in_) { IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen payload input"); + throw detailed_io_error("failed to reopen payload input"); } pay_in_->seek(state.term_state->pay_start); @@ -1612,7 +1613,7 @@ class offs_iterator final : public pos_iterator { if (!pay_in_) { IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen payload input"); + throw detailed_io_error("failed to reopen payload input"); } pay_in_->seek(state.term_state->pay_start); @@ -1722,7 +1723,7 @@ class pay_iterator final : public pos_iterator { if (!pay_in_) { IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen payload input"); + throw detailed_io_error("failed to reopen payload input"); } pay_in_->seek(state.term_state->pay_start); @@ -2042,10 +2043,9 @@ void index_meta_writer::commit() { if (!dir_->rename(src, dst)) { std::stringstream ss; - ss << "Failed to rename file, src path: " << src - << " dst path: " << dst; + ss << "failed to rename file, src path: " << src << " dst path: " << dst; - throw(detailed_io_error(ss.str())); + throw detailed_io_error(ss.str()); } complete(*meta_); @@ -2119,7 +2119,7 @@ void index_meta_reader::read( if (!in) { std::stringstream ss; - ss << "Failed to open file, path: " << meta_file; + ss << "failed to open file, path: " << meta_file; throw detailed_io_error(ss.str()); } @@ -2195,18 +2195,15 @@ void segment_meta_writer::write(directory& dir, const segment_meta& meta) { byte_type flags = meta.column_store ? segment_meta_writer::flags_t::HAS_COLUMN_STORE : 0; if (!out) { - std::stringstream ss; - - ss << "Failed to create file, path: " << meta_file; - - throw detailed_io_error(ss.str()); + throw detailed_io_error("failed to create file, path: " + meta_file); } format_utils::write_header(*out, FORMAT_NAME, FORMAT_MAX); write_string(*out, meta.name); out->write_vlong(meta.version); - out->write_vlong(meta.docs_count); out->write_vlong(meta.live_docs_count); + out->write_vlong(meta.docs_count - meta.live_docs_count); // docs_count >= live_docs_count + out->write_vlong(meta.size); out->write_byte(flags); write_strings( *out, meta.files ); format_utils::write_footer(*out); @@ -2240,7 +2237,7 @@ void segment_meta_reader::read( if (!in) { std::stringstream ss; - ss << "Failed to open file, path: " << meta_file; + ss << "failed to open file, path: " << meta_file; throw detailed_io_error(ss.str()); } @@ -2256,14 +2253,25 @@ void segment_meta_reader::read( auto name = read_string(*in); const auto version = in->read_vlong(); - const auto docs_count = in->read_vlong(); const auto live_docs_count = in->read_vlong(); + const auto docs_count = in->read_vlong() + live_docs_count; + + if (docs_count < live_docs_count) { + throw index_error(std::string("while reader segment meta '") + name + + "', error: docs_count(" + std::to_string(docs_count) + + ") < live_docs_count(" + std::to_string(live_docs_count) + ")" + ); + } + + const auto size = in->read_vlong(); const auto flags = in->read_byte(); auto files = read_strings(*in); if (flags & ~(segment_meta_writer::flags_t::HAS_COLUMN_STORE)) { - // corrupted index - throw index_error(); // use of unsupported flags + throw index_error( + std::string("while reading segment meta '" + name + + "', error: use of unsupported flags '" + std::to_string(flags) + "'") + ); } format_utils::check_footer(*in, checksum); @@ -2277,6 +2285,7 @@ void segment_meta_reader::read( meta.column_store = flags & segment_meta_writer::flags_t::HAS_COLUMN_STORE; meta.docs_count = docs_count; meta.live_docs_count = live_docs_count; + meta.size = size; meta.files = std::move(files); } @@ -2329,11 +2338,7 @@ void document_mask_writer::write( auto out = dir.create(filename); if (!out) { - std::stringstream ss; - - ss << "Failed to create file, path: " << filename; - - throw detailed_io_error(ss.str()); + throw detailed_io_error("Failed to create file, path: " + filename); } // segment can't have more than integer_traits::const_max documents @@ -2342,9 +2347,11 @@ void document_mask_writer::write( format_utils::write_header(*out, FORMAT_NAME, FORMAT_MAX); out->write_vint(count); + for (auto mask : docs_mask) { out->write_vint(mask); } + format_utils::write_footer(*out); } @@ -2678,7 +2685,9 @@ void read_compact( ); if (!irs::type_limits::valid(buf_size)) { - throw irs::index_error(); // corrupted index + throw index_error( + std::string("while reading compact, error: invalid buffer size '") + std::to_string(buf_size) + "'" + ); } } @@ -4993,9 +5002,12 @@ bool postings_reader::prepare( ); const uint64_t block_size = in.read_vint(); + if (block_size != postings_writer::BLOCK_SIZE) { - // invalid block size - throw index_error(); + throw index_error( + std::string("while preparing postings_reader in segment '") + state.meta->name + + "', error: invalid block size '" + std::to_string(block_size) + "'" + ); } return true; diff --git a/3rdParty/iresearch/core/formats/formats_burst_trie.cpp b/3rdParty/iresearch/core/formats/formats_burst_trie.cpp index 021e47b48a..4c8ed6ebd8 100644 --- a/3rdParty/iresearch/core/formats/formats_burst_trie.cpp +++ b/3rdParty/iresearch/core/formats/formats_burst_trie.cpp @@ -179,9 +179,9 @@ inline void prepare_output( if (!out) { std::stringstream ss; - ss << "Failed to create file, path: " << str; + ss << "failed to create file, path: " << str; - throw detailed_io_error(ss.str()); + throw detailed_io_error(ss.str()) ; } format_utils::write_header(*out, format, version); @@ -205,7 +205,7 @@ inline void prepare_input( if (!in) { std::stringstream ss; - ss << "Failed to open file, path: " << str; + ss << "failed to open file, path: " << str; throw detailed_io_error(ss.str()); } @@ -1132,9 +1132,10 @@ index_input& term_iterator::terms_input() const { if (!terms_in_) { IR_FRMT_FATAL("Failed to reopen terms input in: %s", __FUNCTION__); - throw detailed_io_error("Failed to reopen terms input"); + throw detailed_io_error("failed to reopen terms input"); } } + return *terms_in_; } diff --git a/3rdParty/iresearch/core/formats/skip_list.cpp b/3rdParty/iresearch/core/formats/skip_list.cpp index 8a79557c70..d8a009cf5d 100644 --- a/3rdParty/iresearch/core/formats/skip_list.cpp +++ b/3rdParty/iresearch/core/formats/skip_list.cpp @@ -337,10 +337,11 @@ void skip_reader::reset() { void skip_reader::load_level(levels_t& levels, index_input::ptr&& stream, size_t step) { // read level length const auto length = stream->read_vlong(); + if (!length) { - // corrupted index - throw index_error(); + throw index_error("while loading level, error: zero length"); } + const auto begin = stream->file_pointer(); const auto end = begin + length; @@ -384,4 +385,4 @@ NS_END // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/core/index/directory_reader.cpp b/3rdParty/iresearch/core/index/directory_reader.cpp index 81dd72bfa4..21d2cf208b 100644 --- a/3rdParty/iresearch/core/index/directory_reader.cpp +++ b/3rdParty/iresearch/core/index/directory_reader.cpp @@ -321,7 +321,9 @@ directory_reader_impl::directory_reader_impl( } if (!ctx.reader) { - throw index_error(); + throw index_error( + std::string("while opening reader for segment '") + segment.name + "', error: failed to open reader" + ); } ctx.base = static_cast(docs_max); diff --git a/3rdParty/iresearch/core/index/index_writer.cpp b/3rdParty/iresearch/core/index/index_writer.cpp index 424f5d9ff8..372928f366 100644 --- a/3rdParty/iresearch/core/index/index_writer.cpp +++ b/3rdParty/iresearch/core/index/index_writer.cpp @@ -109,7 +109,10 @@ bool add_document_mask_modified_records( auto reader = readers.emplace(meta); if (!reader) { - throw irs::index_error(); // failed to open segment + throw irs::index_error( + std::string("while adding document mask modified records to document_mask of segment '") +meta.name + + "', error: failed to open segment" + ); } bool modified = false; @@ -161,7 +164,10 @@ bool add_document_mask_modified_records( auto reader = readers.emplace(ctx.segment_.meta); if (!reader) { - throw irs::index_error(); // failed to open segment + throw irs::index_error( + std::string("while adding document mask modified records to flush_segment_context of segment '") + ctx.segment_.meta.name + + "', error: failed to open segment" + ); } assert(doc_limits::valid(ctx.doc_id_begin_)); @@ -672,15 +678,19 @@ index_writer::flush_context_ptr index_writer::documents_context::update_segment( || limits.segment_memory_max > writer.memory_active()) // too much memory && !doc_limits::eof(writer.docs_cached())) { // segment full return ctx; - } else { // force a flush of a full segment - IR_FRMT_TRACE( - "Flushing segment '%s', docs=" IR_SIZE_T_SPECIFIER ", memory=" IR_SIZE_T_SPECIFIER ", docs limit=" IR_SIZE_T_SPECIFIER ", memory limit=" IR_SIZE_T_SPECIFIER "", - writer.name().c_str(), writer.docs_cached(), writer.memory_active(), limits.segment_docs_max, limits.segment_memory_max - ); + } - if (!segment.flush()) { - throw index_error(); // failed to flush segment - } + // force a flush of a full segment + IR_FRMT_TRACE( + "Flushing segment '%s', docs=" IR_SIZE_T_SPECIFIER ", memory=" IR_SIZE_T_SPECIFIER ", docs limit=" IR_SIZE_T_SPECIFIER ", memory limit=" IR_SIZE_T_SPECIFIER "", + writer.name().c_str(), writer.docs_cached(), writer.memory_active(), limits.segment_docs_max, limits.segment_memory_max + ); + + if (!segment.flush()) { + throw index_error( + std::string("while flushing segment '") + segment.writer_meta_.meta.name + + "', error: failed to flush segment" + ); } } @@ -2050,7 +2060,7 @@ bool index_writer::start() { auto sync = [&dir](const std::string& file) { if (!dir.sync(file)) { - throw detailed_io_error("Failed to sync file, path: ") << file; + throw detailed_io_error("failed to sync file, path: " + file); } return true; diff --git a/3rdParty/iresearch/core/index/merge_writer.cpp b/3rdParty/iresearch/core/index/merge_writer.cpp index cf9fef30ae..d044c69cdd 100644 --- a/3rdParty/iresearch/core/index/merge_writer.cpp +++ b/3rdParty/iresearch/core/index/merge_writer.cpp @@ -425,10 +425,10 @@ class compound_term_iterator : public irs::term_iterator { ): first(std::move(term_itr)), second(doc_map) { } - // GCC 8.2.0 optimized code requires an *explicit* noexcept non-inlined + // GCC 8.1.0/8.2.0 optimized code requires an *explicit* noexcept non-inlined // move constructor implementation, otherwise the move constructor is fully - // optimized out - GCC8_2_OPTIMIZED_WORKAROUND(__attribute__((noinline))) + // optimized out (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87665) + GCC8_12_OPTIMIZED_WORKAROUND(__attribute__((noinline))) term_iterator_t(term_iterator_t&& other) NOEXCEPT : first(std::move(other.first)), second(std::move(other.second)) { } @@ -1027,4 +1027,4 @@ NS_END // ROOT // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/core/shared.hpp b/3rdParty/iresearch/core/shared.hpp index 46de597e73..9d4dff92ec 100644 --- a/3rdParty/iresearch/core/shared.hpp +++ b/3rdParty/iresearch/core/shared.hpp @@ -154,13 +154,14 @@ #define IMPLICIT_MOVE_WORKAROUND(x) x #endif -// hook for GCC8.2 optimized code +// hook for GCC 8.1/8.2 optimized code // these versions produce incorrect code when inlining optimizations are enabled #if defined(__OPTIMIZE__) && defined(__GNUC__) \ - && (__GNUC__ == 8 && __GNUC_MINOR__ == 2) - #define GCC8_2_OPTIMIZED_WORKAROUND(...) __VA_ARGS__ + && ((__GNUC__ == 8 && __GNUC_MINOR__ == 1) \ + || (__GNUC__ == 8 && __GNUC_MINOR__ == 2)) + #define GCC8_12_OPTIMIZED_WORKAROUND(...) __VA_ARGS__ #else - #define GCC8_2_OPTIMIZED_WORKAROUND(...) + #define GCC8_12_OPTIMIZED_WORKAROUND(...) #endif // hook for MSVC2017.3-8 optimized code diff --git a/3rdParty/iresearch/core/store/directory_attributes.hpp b/3rdParty/iresearch/core/store/directory_attributes.hpp index 189f491f42..950adafa0d 100644 --- a/3rdParty/iresearch/core/store/directory_attributes.hpp +++ b/3rdParty/iresearch/core/store/directory_attributes.hpp @@ -93,6 +93,7 @@ class IRESEARCH_API index_file_refs : public stored_attribute { ref_t add(const std::string& key); ref_t add(std::string&& key); void clear(); + bool remove(const std::string& key) { return refs_.remove(key); } counter_t& refs() NOEXCEPT { return refs_; } @@ -105,4 +106,4 @@ class IRESEARCH_API index_file_refs : public stored_attribute { NS_END -#endif \ No newline at end of file +#endif diff --git a/3rdParty/iresearch/core/store/fs_directory.cpp b/3rdParty/iresearch/core/store/fs_directory.cpp index 795e295e6e..d45c41f989 100644 --- a/3rdParty/iresearch/core/store/fs_directory.cpp +++ b/3rdParty/iresearch/core/store/fs_directory.cpp @@ -37,6 +37,8 @@ #include +#include + NS_LOCAL inline size_t buffer_size(FILE* file) NOEXCEPT { @@ -223,9 +225,11 @@ class fs_index_output : public buffered_index_output { crc.process_bytes(b, len_written); if (len && len_written != len) { - throw detailed_io_error("Failed to write buffer, written ") - << std::to_string(len_written) << " out of " - << std::to_string(len) << " bytes."; + std::stringstream ss; + + ss << "failed to write buffer, written '" << len_written << "' out of '" << len << "' bytes"; + + throw detailed_io_error(ss.str()); } } @@ -328,9 +332,11 @@ class fs_index_input : public buffered_index_input { protected: virtual void seek_internal(size_t pos) override { if (pos >= handle_->size) { - throw detailed_io_error("Seek out of range for input file, length ") - << std::to_string(handle_->size) - << ", position " << std::to_string(pos); + std::stringstream ss; + + ss << "seek out of range for input file, length '" << handle_->size << "', position '" << pos << "'"; + + throw detailed_io_error(ss.str()); } pos_ = pos; @@ -344,9 +350,11 @@ class fs_index_input : public buffered_index_input { if (handle_->pos != pos_) { if (fseek(stream, static_cast(pos_), SEEK_SET) != 0) { - throw detailed_io_error("Failed to seek to ") - << std::to_string(pos_) - << " for input file, error " << std::to_string(ferror(stream)); + std::stringstream ss; + + ss << "failed to seek to '" << pos_ << "' for input file, error '" << ferror(stream) << "'"; + + throw detailed_io_error(ss.str()); } handle_->pos = pos_; @@ -363,10 +371,11 @@ class fs_index_input : public buffered_index_input { } // read error - throw detailed_io_error("Failed to read from input file, read ") - << std::to_string(read) - << " out of " << std::to_string(len) - << " bytes, error " << std::to_string(ferror(stream)); + std::stringstream ss; + + ss << "failed to read from input file, read '" << read << "' out of '" << len << "' bytes, error '" << ferror(stream) << "'"; + + throw detailed_io_error(ss.str()); } assert(handle_->pos == pos_); diff --git a/3rdParty/iresearch/core/utils/compression.cpp b/3rdParty/iresearch/core/utils/compression.cpp index 10b380fe57..816b39de91 100644 --- a/3rdParty/iresearch/core/utils/compression.cpp +++ b/3rdParty/iresearch/core/utils/compression.cpp @@ -74,7 +74,8 @@ void compressor::compress(const char* src, size_t size) { if (lz4_size < 0) { this->size_ = 0; - throw index_error(); // corrupted index + + throw index_error("while compressing, error: LZ4 returned negative size"); } this->data_ = reinterpret_cast(buf); diff --git a/3rdParty/iresearch/core/utils/directory_utils.cpp b/3rdParty/iresearch/core/utils/directory_utils.cpp index 2971c6bc56..80ad39a400 100644 --- a/3rdParty/iresearch/core/utils/directory_utils.cpp +++ b/3rdParty/iresearch/core/utils/directory_utils.cpp @@ -343,16 +343,19 @@ index_input::ptr tracking_directory::open( } bool tracking_directory::remove(const std::string& name) NOEXCEPT { - bool result = impl_.remove(name); + if (!impl_.remove(name)) { + return false; + } try { files_.erase(name); + return true; } catch (...) { IR_LOG_EXCEPTION(); // ignore failure since removal from impl_ was sucessful } - return result; + return false; } bool tracking_directory::rename( @@ -363,9 +366,8 @@ bool tracking_directory::rename( } try { - if (files_.emplace(dst).second) { - files_.erase(src); - } + files_.emplace(dst); + files_.erase(src); return true; } catch (...) { @@ -510,18 +512,23 @@ index_input::ptr ref_tracking_directory::open( } bool ref_tracking_directory::remove(const std::string& name) NOEXCEPT { - bool result = impl_.remove(name); + if (!impl_.remove(name)) { + return false; + } try { + attribute_->remove(name); + SCOPED_LOCK(mutex_); refs_.erase(name); + return true; } catch (...) { IR_LOG_EXCEPTION(); // ignore failure since removal from impl_ was sucessful } - return result; + return false; } bool ref_tracking_directory::rename( @@ -532,15 +539,19 @@ bool ref_tracking_directory::rename( } try { - SCOPED_LOCK(mutex_); + auto ref = attribute_->add(dst); - refs_.emplace(dst, attribute_->add(dst)); - refs_.erase(src); + { + SCOPED_LOCK(mutex_); + refs_.emplace(dst, ref); + refs_.erase(src); + } + + attribute_->remove(src); return true; } catch (...) { IR_LOG_EXCEPTION(); - impl_.rename(dst, src); // revert } return false; diff --git a/3rdParty/iresearch/core/utils/locale_utils.cpp b/3rdParty/iresearch/core/utils/locale_utils.cpp index badd51c961..59faa8bf8d 100644 --- a/3rdParty/iresearch/core/utils/locale_utils.cpp +++ b/3rdParty/iresearch/core/utils/locale_utils.cpp @@ -2684,7 +2684,9 @@ typename num_put_facet::iter_type num_put_facetregular_->format(int64_t(0 - value), ctx->icu_buf0_); if (!converter_.append(ctx->buf_, ctx->icu_buf0_)) { - throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)"); + throw irs::detailed_io_error( + "failed to convert data from UTF8 in num_put_facet::do_put(...)" + ); } size_t len = ctx->buf_.size() + 1; // +1 for '-' @@ -2748,7 +2750,9 @@ typename num_put_facet::iter_type num_put_facet::const_max < value) { - throw irs::detailed_io_error("value too large while converting data from UTF8 in num_put_facet::do_put(...)"); + throw irs::detailed_io_error( + "value too large while converting data from UTF8 in num_put_facet::do_put(...)" + ); } auto ipad = (str.flags() & std::ios_base::adjustfield) == std::ios_base::internal @@ -2765,7 +2769,9 @@ typename num_put_facet::iter_type num_put_facet::iter_type num_put_facetregular_->format(int64_t(value), ctx->icu_buf0_); if (!converter_.append(ctx->buf_, ctx->icu_buf0_)) { - throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)"); + throw irs::detailed_io_error( + "failed to convert data from UTF8 in num_put_facet::do_put(...)" + ); } size_t len = ctx->buf_.size() + (str.flags() & std::ios_base::showpos ? 1 : 0); @@ -2827,7 +2835,9 @@ typename num_put_facet::iter_type num_put_facetreset(str); @@ -2903,7 +2913,9 @@ typename num_put_facet::iter_type num_put_facetbuf_, *icu_buf)) { - throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)"); + throw irs::detailed_io_error( + "failed to convert data from UTF8 in num_put_facet::do_put(...)" + ); } size_t len = ctx->buf_.size() @@ -3759,4 +3771,4 @@ NS_END // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/core/utils/ref_counter.hpp b/3rdParty/iresearch/core/utils/ref_counter.hpp index 97791fd8f5..850fdd715a 100644 --- a/3rdParty/iresearch/core/utils/ref_counter.hpp +++ b/3rdParty/iresearch/core/utils/ref_counter.hpp @@ -56,6 +56,11 @@ class ref_counter : public util::noncopyable { // noncopyable because shared_ptr return itr.first->second; } + bool remove(const Key& key) { + SCOPED_LOCK(lock_); + return refs_.erase(key) > 0; + } + bool contains(const Key& key) const NOEXCEPT { SCOPED_LOCK(lock_); return refs_.find(key) != refs_.end(); diff --git a/3rdParty/iresearch/core/utils/utf8_path.cpp b/3rdParty/iresearch/core/utils/utf8_path.cpp index 91a247a309..9231e54324 100644 --- a/3rdParty/iresearch/core/utils/utf8_path.cpp +++ b/3rdParty/iresearch/core/utils/utf8_path.cpp @@ -155,7 +155,7 @@ utf8_path::utf8_path(const char* utf8_path) utf8_path::utf8_path(const std::string& utf8_path) { if (!append_path(path_, string_ref(utf8_path))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } } @@ -173,7 +173,7 @@ utf8_path::utf8_path(const irs::string_ref& utf8_path) { if (!append_path(path_, utf8_path)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } } @@ -195,14 +195,14 @@ utf8_path::utf8_path(const irs::basic_string_ref& ucs2_path) { if (!append_path(path_, ucs2_path)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } } utf8_path::utf8_path(const std::wstring& ucs2_path) { if (!append_path(path_, wstring_ref(ucs2_path))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } } @@ -213,7 +213,7 @@ utf8_path& utf8_path::operator+=(const char* utf8_name) { utf8_path& utf8_path::operator+=(const std::string &utf8_name) { if (!append_path(path_, string_ref(utf8_name))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -222,7 +222,7 @@ utf8_path& utf8_path::operator+=(const std::string &utf8_name) { utf8_path& utf8_path::operator+=(const string_ref& utf8_name) { if (!append_path(path_, utf8_name)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -235,7 +235,7 @@ utf8_path& utf8_path::operator+=(const wchar_t* ucs2_name) { utf8_path& utf8_path::operator+=(const irs::basic_string_ref& ucs2_name) { if (!append_path(path_, ucs2_name)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -244,7 +244,7 @@ utf8_path& utf8_path::operator+=(const irs::basic_string_ref& ucs2_name utf8_path& utf8_path::operator+=(const std::wstring &ucs2_name) { if (!append_path(path_, wstring_ref(ucs2_name))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -261,7 +261,7 @@ utf8_path& utf8_path::operator/=(const std::string &utf8_name) { if (!append_path(path_, string_ref(utf8_name))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -274,7 +274,7 @@ utf8_path& utf8_path::operator/=(const string_ref& utf8_name) { if (!append_path(path_, utf8_name)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -291,7 +291,7 @@ utf8_path& utf8_path::operator/=(const iresearch::basic_string_ref& ucs if (!append_path(path_, ucs2_name)) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -304,7 +304,7 @@ utf8_path& utf8_path::operator/=(const std::wstring &ucs2_name) { if (!append_path(path_, wstring_ref(ucs2_name))) { // emulate boost::filesystem behaviour by throwing an exception - throw detailed_io_error("Path conversion failure"); + throw detailed_io_error("path conversion failure"); } return *this; @@ -408,4 +408,4 @@ NS_END // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/tests/formats/formats_test_case_base.hpp b/3rdParty/iresearch/tests/formats/formats_test_case_base.hpp index 0b499d2fa9..ff802c4909 100644 --- a/3rdParty/iresearch/tests/formats/formats_test_case_base.hpp +++ b/3rdParty/iresearch/tests/formats/formats_test_case_base.hpp @@ -763,37 +763,75 @@ class format_test_case_base : public index_test_base { } void segment_meta_read_write() { - iresearch::segment_meta meta; - meta.name = "meta_name"; - meta.docs_count = 453; - meta.live_docs_count = 345; - meta.version = 100; - - meta.files.emplace("file1"); - meta.files.emplace("index_file2"); - meta.files.emplace("file3"); - meta.files.emplace("stored_file4"); - - // write segment meta + // read valid meta { - auto writer = codec()->get_segment_meta_writer(); - writer->write(dir(), meta); + iresearch::segment_meta meta; + meta.name = "meta_name"; + meta.docs_count = 453; + meta.live_docs_count = 345; + meta.size = 666; + meta.version = 100; + meta.column_store = true; + + meta.files.emplace("file1"); + meta.files.emplace("index_file2"); + meta.files.emplace("file3"); + meta.files.emplace("stored_file4"); + + // write segment meta + { + auto writer = codec()->get_segment_meta_writer(); + writer->write(dir(), meta); + } + + // read segment meta + { + irs::segment_meta read_meta; + read_meta.name = meta.name; + read_meta.version = 100; + + auto reader = codec()->get_segment_meta_reader(); + reader->read(dir(), read_meta); + ASSERT_EQ(meta.codec, read_meta.codec); // codec stays nullptr + ASSERT_EQ(meta.name, read_meta.name); + ASSERT_EQ(meta.docs_count, read_meta.docs_count); + ASSERT_EQ(meta.live_docs_count, read_meta.live_docs_count); + ASSERT_EQ(meta.version, read_meta.version); + ASSERT_EQ(meta.size, read_meta.size); + ASSERT_EQ(meta.files, read_meta.files); + ASSERT_EQ(meta.column_store, read_meta.column_store); + } } - // read segment meta + // read broken meta (live_docs_count > docs_count) { - irs::segment_meta read_meta; - read_meta.name = meta.name; - read_meta.version = 100; + iresearch::segment_meta meta; + meta.name = "broken_meta_name"; + meta.docs_count = 453; + meta.live_docs_count = 1345; + meta.size = 666; + meta.version = 100; - auto reader = codec()->get_segment_meta_reader(); - reader->read(dir(), read_meta); - ASSERT_EQ(meta.codec, read_meta.codec); // codec stays nullptr - ASSERT_EQ(meta.name, read_meta.name); - ASSERT_EQ(meta.docs_count, read_meta.docs_count); - ASSERT_EQ(meta.live_docs_count, read_meta.live_docs_count); - ASSERT_EQ(meta.version, read_meta.version); - ASSERT_EQ(meta.files, read_meta.files); + meta.files.emplace("file1"); + meta.files.emplace("index_file2"); + meta.files.emplace("file3"); + meta.files.emplace("stored_file4"); + + // write segment meta + { + auto writer = codec()->get_segment_meta_writer(); + writer->write(dir(), meta); + } + + // read segment meta + { + irs::segment_meta read_meta; + read_meta.name = meta.name; + read_meta.version = 100; + + auto reader = codec()->get_segment_meta_reader(); + ASSERT_THROW(reader->read(dir(), read_meta), irs::index_error); + } } } diff --git a/3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp b/3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp index aef53b9dd3..26dfc7a6b2 100644 --- a/3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp +++ b/3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp @@ -24,7 +24,6 @@ #include "tests_shared.hpp" #include "index/index_writer.hpp" -#include "store/memory_directory.hpp" #include "utils/index_utils.hpp" NS_LOCAL @@ -48,7 +47,6 @@ TEST(consolidation_test_tier, test_defaults) { irs::index_utils::consolidate_tier options; auto policy = irs::index_utils::consolidation_policy(options); - irs::memory_directory dir; { irs::index_writer::consolidating_segments_t consolidating_segments; @@ -126,8 +124,6 @@ TEST(consolidation_test_tier, test_defaults) { } TEST(consolidation_test_tier, test_skewed_segments) { - irs::memory_directory dir; - { irs::index_utils::consolidate_tier options; options.min_segments = 1; // min number of segments per tier to merge at once @@ -421,8 +417,48 @@ TEST(consolidation_test_tier, test_skewed_segments) { ASSERT_TRUE(candidates.empty()); } } + // left-skewed distribution + { + const size_t sizes[] = { + 9067, 2228, 9023, 0, 9293, 2637, 7529, 291, 4816, 68, 11, 3582, 4298, 4590, 2772, 9021, 32, 1993, 340, 538, 8578, 258, 8731, 5180, 5708, 339, 3830, 1530, 3906, 8714, 3501, + 1767, 2695, 458, 286, 2506, 3454, 9191, 9368, 305, 17, 219, 6198, 1562, 6303, 7162, 4601, 2687, 8205, 8321, 4568, 2511, 6629, 9109, 9502, 1412, 357, 5235, 137, 9886, 5607, + 1359, 9174, 529, 7074, 8343, 8023, 1618, 6128, 1661, 515, 2388, 2549, 826, 180, 886, 4237, 317, 170, 1532, 1602, 1091, 8953, 1791, 8523, 130, 22, 6319, 6145, 7034, 2006, 52, + 9361, 3443, 8228, 1345, 95, 1940, 6432, 609 + }; + + irs::index_meta meta; + for (auto begin = std::begin(sizes), end = std::end(sizes); begin != end; ++begin) { + const size_t i = std::distance(begin, end); + meta.add(irs::segment_meta(std::to_string(i), nullptr, 100, 100, false, irs::segment_meta::file_set{}, *begin)); + } + + irs::index_utils::consolidate_tier options; + options.min_segments = 10; // min number of segments per tier to merge at once + options.max_segments = 10; // max number of segments per tier to merge at once + options.max_segments_bytes = 250000; // max size of the merge + options.floor_segment_bytes = 50; // smaller segments will be treated as equal to this value + options.lookahead = 0;//irs::integer_traits::const_max; + + auto policy = irs::index_utils::consolidation_policy(options); + + irs::index_writer::consolidating_segments_t consolidating_segments; + std::set candidates; + + do { + candidates.clear(); + policy(candidates, meta, consolidating_segments); + + std::cerr << "Consolidation: "; + for (auto* segment : candidates) { + std::cerr << segment->size << ", "; + } + std::cerr << std::endl; + + consolidating_segments.insert(candidates.begin(), candidates.end()); // register candidates for consolidation + } while (!candidates.empty()); + } } // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/3rdParty/iresearch/tests/index/index_tests.cpp b/3rdParty/iresearch/tests/index/index_tests.cpp index f0118f3db8..b7c12e16a3 100644 --- a/3rdParty/iresearch/tests/index/index_tests.cpp +++ b/3rdParty/iresearch/tests/index/index_tests.cpp @@ -10789,7 +10789,7 @@ class index_test_case_base : public tests::index_test_base { doc4->indexed.begin(), doc4->indexed.end(), doc4->stored.begin(), doc4->stored.end() )); - ASSERT_THROW(writer->commit(), irs::io_error); + ASSERT_THROW(writer->commit(), irs::detailed_io_error); } // check index, it should be empty diff --git a/arangod/IResearch/IResearchView.cpp b/arangod/IResearch/IResearchView.cpp index 97650f04da..8b4cea543c 100644 --- a/arangod/IResearch/IResearchView.cpp +++ b/arangod/IResearch/IResearchView.cpp @@ -1543,8 +1543,14 @@ int IResearchView::insert( || !impl.updateProperties(meta).ok() // update separately since per-instance async jobs already started || !impl._metaState.init(properties, error)) { TRI_set_errno(TRI_ERROR_BAD_PARAMETER); - LOG_TOPIC(WARN, arangodb::iresearch::TOPIC) - << "failed to initialize arangosearch view from definition, error: " << error; + + if (error.empty()) { + LOG_TOPIC(WARN, arangodb::iresearch::TOPIC) + << "failed to initialize arangosearch view '" << impl.name() << "' from definition"; + } else { + LOG_TOPIC(WARN, arangodb::iresearch::TOPIC) + << "failed to initialize arangosearch view '" << impl.name() << "' from definition, error in attribute: " << error; + } return nullptr; } @@ -1928,7 +1934,12 @@ arangodb::Result IResearchView::updateProperties( auto& initialMeta = partialUpdate ? *metaPtr : IResearchViewMeta::DEFAULT(); if (!meta.init(slice, error, initialMeta)) { - return arangodb::Result(TRI_ERROR_BAD_PARAMETER, std::move(error)); + return arangodb::Result( + TRI_ERROR_BAD_PARAMETER, + error.empty() + ? (std::string("failed to update arangosearch view '") + name() + "' from definition") + : (std::string("failed to update arangosearch view '") + name() + "' from definition, error in attribute: " + error) + ); } // reset non-updatable values to match current meta diff --git a/arangod/RestHandler/RestViewHandler.cpp b/arangod/RestHandler/RestViewHandler.cpp index 932524ff25..93f1774641 100644 --- a/arangod/RestHandler/RestViewHandler.cpp +++ b/arangod/RestHandler/RestViewHandler.cpp @@ -345,7 +345,7 @@ void RestViewHandler::modifyView(bool partialUpdate) { ); // TODO: not force sync? if (!result.ok()) { - generateError(GeneralResponse::responseCode(result.errorNumber()), result.errorNumber(), result.errorMessage()); + generateError(result); return; }