Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

issue 2921: make error message more detailed (#7004) 

* issue 2921: make error message more detailed

* upgrade iresearch
This commit is contained in:
Vasiliy 2018-10-23 01:06:44 +03:00 committed by Andrey Abramov
parent 806e7d0f1f
commit 5d2062ffbe
23 changed files with 367 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
3rdParty/iresearch/core/error/error.cpp vendored
View File

@ -74,52 +74,6 @@ const char* eof_error::what() const NOEXCEPT {
return "Read past EOF.";
}
// ----------------------------------------------------------------------------
// detailed_io_error
// ----------------------------------------------------------------------------
detailed_io_error::detailed_io_error(
const irs::string_ref& error /*= irs::string_ref::NIL*/
) {
if (!error.empty()) {
error_.append(error.c_str(), error.size());
}
}
detailed_io_error::detailed_io_error(std::string&& error)
: error_(std::move(error)) {
}
detailed_io_error::detailed_io_error(const char* error)
: detailed_io_error(irs::string_ref(error)) {
}
detailed_io_error& detailed_io_error::operator<<(const irs::string_ref& error) {
if (!error.empty()) {
error_.append(error.c_str(), error.size());
}
return *this;
}
detailed_io_error& detailed_io_error::operator<<(std::string&& error) {
error.append(std::move(error));
return *this;
}
detailed_io_error& detailed_io_error::operator<<(const char* error) {
return (*this) << irs::string_ref(error);
}
ErrorCode detailed_io_error::code() const NOEXCEPT {
return CODE;
}
const char* detailed_io_error::what() const NOEXCEPT {
return error_.c_str();
}
// ----------------------------------------------------------------------------
// lock_obtain_failed
// ----------------------------------------------------------------------------
@ -178,18 +132,6 @@ const char* index_not_found::what() const NOEXCEPT {
return "No segments* file found.";
}
// ----------------------------------------------------------------------------
// index_error
// ----------------------------------------------------------------------------
ErrorCode index_error::code() const NOEXCEPT{
return CODE;
}
const char* index_error::what() const NOEXCEPT {
return "Index error.";
}
// ----------------------------------------------------------------------------
// not_impl_error
// ----------------------------------------------------------------------------

64
3rdParty/iresearch/core/error/error.hpp vendored
View File

@ -24,10 +24,10 @@
#ifndef IRESEARCH_ERROR_H
#define IRESEARCH_ERROR_H
#include "utils/string.hpp"
#include <exception>
#include "utils/string.hpp"
MSVC_ONLY(class IRESEARCH_API std::exception);
NS_ROOT
@ -57,6 +57,34 @@ struct IRESEARCH_API error_base: std::exception {
virtual const char* what() const NOEXCEPT override;
};
// -----------------------------------------------------------------------------
// detailed_error_base
// -----------------------------------------------------------------------------
class IRESEARCH_API detailed_error_base: public error_base {
public:
explicit detailed_error_base(std::string&& error) NOEXCEPT
: error_(std::move(error)) {
}
detailed_error_base(const detailed_error_base& other)
: error_(other.error_) {
}
detailed_error_base(detailed_error_base&& other) NOEXCEPT
: error_(std::move(other.error_)) {
}
detailed_error_base& operator=(const detailed_error_base& other) = delete;
detailed_error_base& operator=(detailed_error_base&& other) NOEXCEPT = delete;
virtual const char* what() const NOEXCEPT final { return error_.c_str(); }
private:
IRESEARCH_API_PRIVATE_VARIABLES_BEGIN
std::string error_;
IRESEARCH_API_PRIVATE_VARIABLES_END
};
// ----------------------------------------------------------------------------
// not_supported
// ----------------------------------------------------------------------------
@ -87,21 +115,14 @@ struct IRESEARCH_API eof_error: io_error {
// ----------------------------------------------------------------------------
// detailed_io_error
// ----------------------------------------------------------------------------
class IRESEARCH_API detailed_io_error: public io_error {
struct IRESEARCH_API detailed_io_error: public detailed_error_base {
public:
explicit detailed_io_error(std::string&& error)
: detailed_error_base(std::move(error)) {
}
DECLARE_ERROR_CODE(io_error);
explicit detailed_io_error(const irs::string_ref& error = irs::string_ref::NIL);
explicit detailed_io_error(std::string&& error);
explicit detailed_io_error(const char* error);
detailed_io_error& operator<<(const irs::string_ref& error);
detailed_io_error& operator<<(std::string&& error);
detailed_io_error& operator<<(const char* error);
virtual iresearch::ErrorCode code() const NOEXCEPT override;
virtual const char* what() const NOEXCEPT override;
private:
IRESEARCH_API_PRIVATE_VARIABLES_BEGIN
std::string error_;
IRESEARCH_API_PRIVATE_VARIABLES_END
virtual ErrorCode code() const NOEXCEPT override { return CODE; }
};
// ----------------------------------------------------------------------------
@ -146,10 +167,13 @@ struct IRESEARCH_API index_not_found: error_base {
// ----------------------------------------------------------------------------
// index_error
// ----------------------------------------------------------------------------
struct IRESEARCH_API index_error: error_base {
DECLARE_ERROR_CODE( index_error );
virtual ErrorCode code() const NOEXCEPT override;
virtual const char* what() const NOEXCEPT override;
struct IRESEARCH_API index_error: detailed_error_base {
explicit index_error(std::string&& error)
: detailed_error_base(std::move(error)) {
}
DECLARE_ERROR_CODE(index_error);
virtual ErrorCode code() const NOEXCEPT override { return CODE; }
};
// ----------------------------------------------------------------------------
@ -181,4 +205,4 @@ struct IRESEARCH_API illegal_state: error_base {
NS_END
#endif
#endif

41
3rdParty/iresearch/core/formats/format_utils.cpp vendored
View File

@ -28,25 +28,33 @@
#include "formats/formats.hpp"
#include <sstream>
NS_ROOT
void validate_footer(iresearch::index_input& in) {
const int64_t remain = in.length() - in.file_pointer();
if (remain != format_utils::FOOTER_LEN) {
// invalid position
throw iresearch::index_error();
throw index_error(
std::string("while validating footer, error: invalid position '") + std::to_string(remain) + "'"
);
}
const int32_t magic = in.read_int();
if (magic != format_utils::FOOTER_MAGIC) {
// invalid magic number
throw iresearch::index_error();
throw index_error(
std::string("while validating footer, error: invalid magic number '") + std::to_string(magic) + "'"
);
}
const int32_t alg_id = in.read_int();
if (alg_id != 0) {
// invalid algorithm
throw iresearch::index_error();
throw index_error(
std::string("while validating footer, error: invalid algorithm '") + std::to_string(alg_id) + "'"
);
}
}
@ -69,25 +77,34 @@ int32_t check_header(
const string_ref& req_format,
int32_t min_ver, int32_t max_ver) {
const int32_t magic = in.read_int();
if (FORMAT_MAGIC != magic) {
// index format
throw index_error();
throw irs::index_error(
std::string("while checking header, error: invalid magic '") + std::to_string(magic) + "'"
);
}
const auto format = read_string<std::string>(in);
if (compare(req_format, format) != 0) {
// invalid format
throw index_error();
std::stringstream ss;
ss << "while checking header, error: format mismatch '" << format << "' != '" << req_format << "'";
throw irs::index_error(ss.str());
}
const int32_t ver = in.read_int();
if (ver < min_ver || ver > max_ver) {
// invalid version
throw index_error();
throw irs::index_error(
std::string("while checking header, error: invalid version '") + std::to_string(ver) + "'"
);
}
return ver;
}
NS_END
NS_END

5
3rdParty/iresearch/core/formats/format_utils.hpp vendored
View File

@ -81,8 +81,9 @@ inline int64_t check_footer(index_input& in, int64_t checksum) {
validate_footer(in);
if (checksum != in.read_long()) {
// invalid checksum
throw index_error();
throw index_error(
std::string("while checking footer, error: invalid checksum '") + std::to_string(checksum) + "'"
);
}
return checksum;

74
3rdParty/iresearch/core/formats/formats_10.cpp vendored
View File

@ -217,7 +217,7 @@ inline void prepare_output(
if (!out) {
std::stringstream ss;
ss << "Failed to create file, path: " << str;
ss << "failed to create file, path: " << str;
throw detailed_io_error(ss.str());
}
@ -241,7 +241,7 @@ inline void prepare_input(
if (!in) {
std::stringstream ss;
ss << "Failed to open file, path: " << str;
ss << "failed to open file, path: " << str;
throw detailed_io_error(ss.str());
}
@ -652,8 +652,9 @@ void postings_writer::begin_doc(doc_id_t id, const frequency* freq) {
}
if (id < doc.last) {
// docs out of order
throw index_error();
throw index_error(
std::string("while beginning doc in postings_writer, error: docs out of order '") + std::to_string(id) + "' < '" + std::to_string(doc.last) + "'"
);
}
doc.doc(id - doc.last);
@ -999,7 +1000,7 @@ class doc_iterator : public iresearch::doc_iterator {
if (!doc_in_) {
IR_FRMT_FATAL("Failed to reopen document input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen document input");
throw detailed_io_error("failed to reopen document input");
}
}
@ -1354,7 +1355,7 @@ class pos_iterator: public position {
if (!pos_in_) {
IR_FRMT_FATAL("Failed to reopen positions input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen positions input");
throw detailed_io_error("failed to reopen positions input");
}
pos_in_->seek(state.term_state->pos_start);
@ -1467,7 +1468,7 @@ class offs_pay_iterator final: public pos_iterator {
if (!pay_in_) {
IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen payload input");
throw detailed_io_error("failed to reopen payload input");
}
pay_in_->seek(state.term_state->pay_start);
@ -1612,7 +1613,7 @@ class offs_iterator final : public pos_iterator {
if (!pay_in_) {
IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen payload input");
throw detailed_io_error("failed to reopen payload input");
}
pay_in_->seek(state.term_state->pay_start);
@ -1722,7 +1723,7 @@ class pay_iterator final : public pos_iterator {
if (!pay_in_) {
IR_FRMT_FATAL("Failed to reopen payload input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen payload input");
throw detailed_io_error("failed to reopen payload input");
}
pay_in_->seek(state.term_state->pay_start);
@ -2042,10 +2043,9 @@ void index_meta_writer::commit() {
if (!dir_->rename(src, dst)) {
std::stringstream ss;
ss << "Failed to rename file, src path: " << src
<< " dst path: " << dst;
ss << "failed to rename file, src path: " << src << " dst path: " << dst;
throw(detailed_io_error(ss.str()));
throw detailed_io_error(ss.str());
}
complete(*meta_);
@ -2119,7 +2119,7 @@ void index_meta_reader::read(
if (!in) {
std::stringstream ss;
ss << "Failed to open file, path: " << meta_file;
ss << "failed to open file, path: " << meta_file;
throw detailed_io_error(ss.str());
}
@ -2195,18 +2195,15 @@ void segment_meta_writer::write(directory& dir, const segment_meta& meta) {
byte_type flags = meta.column_store ? segment_meta_writer::flags_t::HAS_COLUMN_STORE : 0;
if (!out) {
std::stringstream ss;
ss << "Failed to create file, path: " << meta_file;
throw detailed_io_error(ss.str());
throw detailed_io_error("failed to create file, path: " + meta_file);
}
format_utils::write_header(*out, FORMAT_NAME, FORMAT_MAX);
write_string(*out, meta.name);
out->write_vlong(meta.version);
out->write_vlong(meta.docs_count);
out->write_vlong(meta.live_docs_count);
out->write_vlong(meta.docs_count - meta.live_docs_count); // docs_count >= live_docs_count
out->write_vlong(meta.size);
out->write_byte(flags);
write_strings( *out, meta.files );
format_utils::write_footer(*out);
@ -2240,7 +2237,7 @@ void segment_meta_reader::read(
if (!in) {
std::stringstream ss;
ss << "Failed to open file, path: " << meta_file;
ss << "failed to open file, path: " << meta_file;
throw detailed_io_error(ss.str());
}
@ -2256,14 +2253,25 @@ void segment_meta_reader::read(
auto name = read_string<std::string>(*in);
const auto version = in->read_vlong();
const auto docs_count = in->read_vlong();
const auto live_docs_count = in->read_vlong();
const auto docs_count = in->read_vlong() + live_docs_count;
if (docs_count < live_docs_count) {
throw index_error(std::string("while reader segment meta '") + name
+ "', error: docs_count(" + std::to_string(docs_count)
+ ") < live_docs_count(" + std::to_string(live_docs_count) + ")"
);
}
const auto size = in->read_vlong();
const auto flags = in->read_byte();
auto files = read_strings<segment_meta::file_set>(*in);
if (flags & ~(segment_meta_writer::flags_t::HAS_COLUMN_STORE)) {
// corrupted index
throw index_error(); // use of unsupported flags
throw index_error(
std::string("while reading segment meta '" + name
+ "', error: use of unsupported flags '" + std::to_string(flags) + "'")
);
}
format_utils::check_footer(*in, checksum);
@ -2277,6 +2285,7 @@ void segment_meta_reader::read(
meta.column_store = flags & segment_meta_writer::flags_t::HAS_COLUMN_STORE;
meta.docs_count = docs_count;
meta.live_docs_count = live_docs_count;
meta.size = size;
meta.files = std::move(files);
}
@ -2329,11 +2338,7 @@ void document_mask_writer::write(
auto out = dir.create(filename);
if (!out) {
std::stringstream ss;
ss << "Failed to create file, path: " << filename;
throw detailed_io_error(ss.str());
throw detailed_io_error("Failed to create file, path: " + filename);
}
// segment can't have more than integer_traits<uint32_t>::const_max documents
@ -2342,9 +2347,11 @@ void document_mask_writer::write(
format_utils::write_header(*out, FORMAT_NAME, FORMAT_MAX);
out->write_vint(count);
for (auto mask : docs_mask) {
out->write_vint(mask);
}
format_utils::write_footer(*out);
}
@ -2678,7 +2685,9 @@ void read_compact(
);
if (!irs::type_limits<iresearch::type_t::address_t>::valid(buf_size)) {
throw irs::index_error(); // corrupted index
throw index_error(
std::string("while reading compact, error: invalid buffer size '") + std::to_string(buf_size) + "'"
);
}
}
@ -4993,9 +5002,12 @@ bool postings_reader::prepare(
);
const uint64_t block_size = in.read_vint();
if (block_size != postings_writer::BLOCK_SIZE) {
// invalid block size
throw index_error();
throw index_error(
std::string("while preparing postings_reader in segment '") + state.meta->name
+ "', error: invalid block size '" + std::to_string(block_size) + "'"
);
}
return true;

9
3rdParty/iresearch/core/formats/formats_burst_trie.cpp vendored
View File

@ -179,9 +179,9 @@ inline void prepare_output(
if (!out) {
std::stringstream ss;
ss << "Failed to create file, path: " << str;
ss << "failed to create file, path: " << str;
throw detailed_io_error(ss.str());
throw detailed_io_error(ss.str()) ;
}
format_utils::write_header(*out, format, version);
@ -205,7 +205,7 @@ inline void prepare_input(
if (!in) {
std::stringstream ss;
ss << "Failed to open file, path: " << str;
ss << "failed to open file, path: " << str;
throw detailed_io_error(ss.str());
}
@ -1132,9 +1132,10 @@ index_input& term_iterator::terms_input() const {
if (!terms_in_) {
IR_FRMT_FATAL("Failed to reopen terms input in: %s", __FUNCTION__);
throw detailed_io_error("Failed to reopen terms input");
throw detailed_io_error("failed to reopen terms input");
}
}
return *terms_in_;
}

7
3rdParty/iresearch/core/formats/skip_list.cpp vendored
View File

@ -337,10 +337,11 @@ void skip_reader::reset() {
void skip_reader::load_level(levels_t& levels, index_input::ptr&& stream, size_t step) {
// read level length
const auto length = stream->read_vlong();
if (!length) {
// corrupted index
throw index_error();
throw index_error("while loading level, error: zero length");
}
const auto begin = stream->file_pointer();
const auto end = begin + length;
@ -384,4 +385,4 @@ NS_END
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

4
3rdParty/iresearch/core/index/directory_reader.cpp vendored
View File

@ -321,7 +321,9 @@ directory_reader_impl::directory_reader_impl(
}
if (!ctx.reader) {
throw index_error();
throw index_error(
std::string("while opening reader for segment '") + segment.name + "', error: failed to open reader"
);
}
ctx.base = static_cast<doc_id_t>(docs_max);

32
3rdParty/iresearch/core/index/index_writer.cpp vendored
View File

@ -109,7 +109,10 @@ bool add_document_mask_modified_records(
auto reader = readers.emplace(meta);
if (!reader) {
throw irs::index_error(); // failed to open segment
throw irs::index_error(
std::string("while adding document mask modified records to document_mask of segment '") +meta.name
+ "', error: failed to open segment"
);
}
bool modified = false;
@ -161,7 +164,10 @@ bool add_document_mask_modified_records(
auto reader = readers.emplace(ctx.segment_.meta);
if (!reader) {
throw irs::index_error(); // failed to open segment
throw irs::index_error(
std::string("while adding document mask modified records to flush_segment_context of segment '") + ctx.segment_.meta.name
+ "', error: failed to open segment"
);
}
assert(doc_limits::valid(ctx.doc_id_begin_));
@ -672,15 +678,19 @@ index_writer::flush_context_ptr index_writer::documents_context::update_segment(
|| limits.segment_memory_max > writer.memory_active()) // too much memory
&& !doc_limits::eof(writer.docs_cached())) { // segment full
return ctx;
} else { // force a flush of a full segment
IR_FRMT_TRACE(
"Flushing segment '%s', docs=" IR_SIZE_T_SPECIFIER ", memory=" IR_SIZE_T_SPECIFIER ", docs limit=" IR_SIZE_T_SPECIFIER ", memory limit=" IR_SIZE_T_SPECIFIER "",
writer.name().c_str(), writer.docs_cached(), writer.memory_active(), limits.segment_docs_max, limits.segment_memory_max
);
}
if (!segment.flush()) {
throw index_error(); // failed to flush segment
}
// force a flush of a full segment
IR_FRMT_TRACE(
"Flushing segment '%s', docs=" IR_SIZE_T_SPECIFIER ", memory=" IR_SIZE_T_SPECIFIER ", docs limit=" IR_SIZE_T_SPECIFIER ", memory limit=" IR_SIZE_T_SPECIFIER "",
writer.name().c_str(), writer.docs_cached(), writer.memory_active(), limits.segment_docs_max, limits.segment_memory_max
);
if (!segment.flush()) {
throw index_error(
std::string("while flushing segment '") + segment.writer_meta_.meta.name
+ "', error: failed to flush segment"
);
}
}
@ -2050,7 +2060,7 @@ bool index_writer::start() {
auto sync = [&dir](const std::string& file) {
if (!dir.sync(file)) {
throw detailed_io_error("Failed to sync file, path: ") << file;
throw detailed_io_error("failed to sync file, path: " + file);
}
return true;

8
3rdParty/iresearch/core/index/merge_writer.cpp vendored
View File

@ -425,10 +425,10 @@ class compound_term_iterator : public irs::term_iterator {
): first(std::move(term_itr)), second(doc_map) {
}
// GCC 8.2.0 optimized code requires an *explicit* noexcept non-inlined
// GCC 8.1.0/8.2.0 optimized code requires an *explicit* noexcept non-inlined
// move constructor implementation, otherwise the move constructor is fully
// optimized out
GCC8_2_OPTIMIZED_WORKAROUND(__attribute__((noinline)))
// optimized out (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87665)
GCC8_12_OPTIMIZED_WORKAROUND(__attribute__((noinline)))
term_iterator_t(term_iterator_t&& other) NOEXCEPT
: first(std::move(other.first)), second(std::move(other.second)) {
}
@ -1027,4 +1027,4 @@ NS_END // ROOT
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

9
3rdParty/iresearch/core/shared.hpp vendored
View File

@ -154,13 +154,14 @@
#define IMPLICIT_MOVE_WORKAROUND(x) x
#endif
// hook for GCC8.2 optimized code
// hook for GCC 8.1/8.2 optimized code
// these versions produce incorrect code when inlining optimizations are enabled
#if defined(__OPTIMIZE__) && defined(__GNUC__) \
&& (__GNUC__ == 8 && __GNUC_MINOR__ == 2)
#define GCC8_2_OPTIMIZED_WORKAROUND(...) __VA_ARGS__
&& ((__GNUC__ == 8 && __GNUC_MINOR__ == 1) \
|| (__GNUC__ == 8 && __GNUC_MINOR__ == 2))
#define GCC8_12_OPTIMIZED_WORKAROUND(...) __VA_ARGS__
#else
#define GCC8_2_OPTIMIZED_WORKAROUND(...)
#define GCC8_12_OPTIMIZED_WORKAROUND(...)
#endif
// hook for MSVC2017.3-8 optimized code

3
3rdParty/iresearch/core/store/directory_attributes.hpp vendored
View File

@ -93,6 +93,7 @@ class IRESEARCH_API index_file_refs : public stored_attribute {
ref_t add(const std::string& key);
ref_t add(std::string&& key);
void clear();
bool remove(const std::string& key) { return refs_.remove(key); }
counter_t& refs() NOEXCEPT {
return refs_;
}
@ -105,4 +106,4 @@ class IRESEARCH_API index_file_refs : public stored_attribute {
NS_END
#endif
#endif

35
3rdParty/iresearch/core/store/fs_directory.cpp vendored
View File

@ -37,6 +37,8 @@
#include <boost/locale/encoding.hpp>
#include <sstream>
NS_LOCAL
inline size_t buffer_size(FILE* file) NOEXCEPT {
@ -223,9 +225,11 @@ class fs_index_output : public buffered_index_output {
crc.process_bytes(b, len_written);
if (len && len_written != len) {
throw detailed_io_error("Failed to write buffer, written ")
<< std::to_string(len_written) << " out of "
<< std::to_string(len) << " bytes.";
std::stringstream ss;
ss << "failed to write buffer, written '" << len_written << "' out of '" << len << "' bytes";
throw detailed_io_error(ss.str());
}
}
@ -328,9 +332,11 @@ class fs_index_input : public buffered_index_input {
protected:
virtual void seek_internal(size_t pos) override {
if (pos >= handle_->size) {
throw detailed_io_error("Seek out of range for input file, length ")
<< std::to_string(handle_->size)
<< ", position " << std::to_string(pos);
std::stringstream ss;
ss << "seek out of range for input file, length '" << handle_->size << "', position '" << pos << "'";
throw detailed_io_error(ss.str());
}
pos_ = pos;
@ -344,9 +350,11 @@ class fs_index_input : public buffered_index_input {
if (handle_->pos != pos_) {
if (fseek(stream, static_cast<long>(pos_), SEEK_SET) != 0) {
throw detailed_io_error("Failed to seek to ")
<< std::to_string(pos_)
<< " for input file, error " << std::to_string(ferror(stream));
std::stringstream ss;
ss << "failed to seek to '" << pos_ << "' for input file, error '" << ferror(stream) << "'";
throw detailed_io_error(ss.str());
}
handle_->pos = pos_;
@ -363,10 +371,11 @@ class fs_index_input : public buffered_index_input {
}
// read error
throw detailed_io_error("Failed to read from input file, read ")
<< std::to_string(read)
<< " out of " << std::to_string(len)
<< " bytes, error " << std::to_string(ferror(stream));
std::stringstream ss;
ss << "failed to read from input file, read '" << read << "' out of '" << len << "' bytes, error '" << ferror(stream) << "'";
throw detailed_io_error(ss.str());
}
assert(handle_->pos == pos_);

3
3rdParty/iresearch/core/utils/compression.cpp vendored
View File

@ -74,7 +74,8 @@ void compressor::compress(const char* src, size_t size) {
if (lz4_size < 0) {
this->size_ = 0;
throw index_error(); // corrupted index
throw index_error("while compressing, error: LZ4 returned negative size");
}
this->data_ = reinterpret_cast<const byte_type*>(buf);

33
3rdParty/iresearch/core/utils/directory_utils.cpp vendored
View File

@ -343,16 +343,19 @@ index_input::ptr tracking_directory::open(
}
bool tracking_directory::remove(const std::string& name) NOEXCEPT {
bool result = impl_.remove(name);
if (!impl_.remove(name)) {
return false;
}
try {
files_.erase(name);
return true;
} catch (...) {
IR_LOG_EXCEPTION();
// ignore failure since removal from impl_ was sucessful
}
return result;
return false;
}
bool tracking_directory::rename(
@ -363,9 +366,8 @@ bool tracking_directory::rename(
}
try {
if (files_.emplace(dst).second) {
files_.erase(src);
}
files_.emplace(dst);
files_.erase(src);
return true;
} catch (...) {
@ -510,18 +512,23 @@ index_input::ptr ref_tracking_directory::open(
}
bool ref_tracking_directory::remove(const std::string& name) NOEXCEPT {
bool result = impl_.remove(name);
if (!impl_.remove(name)) {
return false;
}
try {
attribute_->remove(name);
SCOPED_LOCK(mutex_);
refs_.erase(name);
return true;
} catch (...) {
IR_LOG_EXCEPTION();
// ignore failure since removal from impl_ was sucessful
}
return result;
return false;
}
bool ref_tracking_directory::rename(
@ -532,15 +539,19 @@ bool ref_tracking_directory::rename(
}
try {
SCOPED_LOCK(mutex_);
auto ref = attribute_->add(dst);
refs_.emplace(dst, attribute_->add(dst));
refs_.erase(src);
{
SCOPED_LOCK(mutex_);
refs_.emplace(dst, ref);
refs_.erase(src);
}
attribute_->remove(src);
return true;
} catch (...) {
IR_LOG_EXCEPTION();
impl_.rename(dst, src); // revert
}
return false;

26
3rdParty/iresearch/core/utils/locale_utils.cpp vendored
View File

@ -2684,7 +2684,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
ctx->regular_->format(int64_t(0 - value), ctx->icu_buf0_);
if (!converter_.append(ctx->buf_, ctx->icu_buf0_)) {
throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"failed to convert data from UTF8 in num_put_facet::do_put(...)"
);
}
size_t len = ctx->buf_.size() + 1; // +1 for '-'
@ -2748,7 +2750,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
}
if ((unsigned long long)irs::integer_traits<int64_t>::const_max < value) {
throw irs::detailed_io_error("value too large while converting data from UTF8 in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"value too large while converting data from UTF8 in num_put_facet::do_put(...)"
);
}
auto ipad = (str.flags() & std::ios_base::adjustfield) == std::ios_base::internal
@ -2765,7 +2769,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
auto ctx = context();
if (!ctx) {
throw irs::detailed_io_error("failed to retrieve ICU formatter in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"failed to retrieve ICU formatter in num_put_facet::do_put(...)"
);
}
static_assert(sizeof(int64_t) == sizeof(long long), "sizeof(int64_t) != sizeof(long long)");
@ -2773,7 +2779,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
ctx->regular_->format(int64_t(value), ctx->icu_buf0_);
if (!converter_.append(ctx->buf_, ctx->icu_buf0_)) {
throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"failed to convert data from UTF8 in num_put_facet::do_put(...)"
);
}
size_t len = ctx->buf_.size() + (str.flags() & std::ios_base::showpos ? 1 : 0);
@ -2827,7 +2835,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
auto ctx = context();
if (!ctx) {
throw irs::detailed_io_error("failed to retrieve ICU formatter in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"failed to retrieve ICU formatter in num_put_facet::do_put(...)"
);
}
ctx->reset(str);
@ -2903,7 +2913,9 @@ typename num_put_facet<CharType, CvtType>::iter_type num_put_facet<CharType, Cvt
}
if (!converter_.append(ctx->buf_, *icu_buf)) {
throw irs::detailed_io_error("failed to convert data from UTF8 in num_put_facet::do_put(...)");
throw irs::detailed_io_error(
"failed to convert data from UTF8 in num_put_facet::do_put(...)"
);
}
size_t len = ctx->buf_.size()
@ -3759,4 +3771,4 @@ NS_END
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

5
3rdParty/iresearch/core/utils/ref_counter.hpp vendored
View File

@ -56,6 +56,11 @@ class ref_counter : public util::noncopyable { // noncopyable because shared_ptr
return itr.first->second;
}
bool remove(const Key& key) {
SCOPED_LOCK(lock_);
return refs_.erase(key) > 0;
}
bool contains(const Key& key) const NOEXCEPT {
SCOPED_LOCK(lock_);
return refs_.find(key) != refs_.end();

26
3rdParty/iresearch/core/utils/utf8_path.cpp vendored
View File

@ -155,7 +155,7 @@ utf8_path::utf8_path(const char* utf8_path)
utf8_path::utf8_path(const std::string& utf8_path) {
if (!append_path(path_, string_ref(utf8_path))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
}
@ -173,7 +173,7 @@ utf8_path::utf8_path(const irs::string_ref& utf8_path) {
if (!append_path(path_, utf8_path)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
}
@ -195,14 +195,14 @@ utf8_path::utf8_path(const irs::basic_string_ref<wchar_t>& ucs2_path) {
if (!append_path(path_, ucs2_path)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
}
utf8_path::utf8_path(const std::wstring& ucs2_path) {
if (!append_path(path_, wstring_ref(ucs2_path))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
}
@ -213,7 +213,7 @@ utf8_path& utf8_path::operator+=(const char* utf8_name) {
utf8_path& utf8_path::operator+=(const std::string &utf8_name) {
if (!append_path(path_, string_ref(utf8_name))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -222,7 +222,7 @@ utf8_path& utf8_path::operator+=(const std::string &utf8_name) {
utf8_path& utf8_path::operator+=(const string_ref& utf8_name) {
if (!append_path(path_, utf8_name)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -235,7 +235,7 @@ utf8_path& utf8_path::operator+=(const wchar_t* ucs2_name) {
utf8_path& utf8_path::operator+=(const irs::basic_string_ref<wchar_t>& ucs2_name) {
if (!append_path(path_, ucs2_name)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -244,7 +244,7 @@ utf8_path& utf8_path::operator+=(const irs::basic_string_ref<wchar_t>& ucs2_name
utf8_path& utf8_path::operator+=(const std::wstring &ucs2_name) {
if (!append_path(path_, wstring_ref(ucs2_name))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -261,7 +261,7 @@ utf8_path& utf8_path::operator/=(const std::string &utf8_name) {
if (!append_path(path_, string_ref(utf8_name))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -274,7 +274,7 @@ utf8_path& utf8_path::operator/=(const string_ref& utf8_name) {
if (!append_path(path_, utf8_name)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -291,7 +291,7 @@ utf8_path& utf8_path::operator/=(const iresearch::basic_string_ref<wchar_t>& ucs
if (!append_path(path_, ucs2_name)) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -304,7 +304,7 @@ utf8_path& utf8_path::operator/=(const std::wstring &ucs2_name) {
if (!append_path(path_, wstring_ref(ucs2_name))) {
// emulate boost::filesystem behaviour by throwing an exception
throw detailed_io_error("Path conversion failure");
throw detailed_io_error("path conversion failure");
}
return *this;
@ -408,4 +408,4 @@ NS_END
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

90
3rdParty/iresearch/tests/formats/formats_test_case_base.hpp vendored
View File

@ -763,37 +763,75 @@ class format_test_case_base : public index_test_base {
}
void segment_meta_read_write() {
iresearch::segment_meta meta;
meta.name = "meta_name";
meta.docs_count = 453;
meta.live_docs_count = 345;
meta.version = 100;
meta.files.emplace("file1");
meta.files.emplace("index_file2");
meta.files.emplace("file3");
meta.files.emplace("stored_file4");
// write segment meta
// read valid meta
{
auto writer = codec()->get_segment_meta_writer();
writer->write(dir(), meta);
iresearch::segment_meta meta;
meta.name = "meta_name";
meta.docs_count = 453;
meta.live_docs_count = 345;
meta.size = 666;
meta.version = 100;
meta.column_store = true;
meta.files.emplace("file1");
meta.files.emplace("index_file2");
meta.files.emplace("file3");
meta.files.emplace("stored_file4");
// write segment meta
{
auto writer = codec()->get_segment_meta_writer();
writer->write(dir(), meta);
}
// read segment meta
{
irs::segment_meta read_meta;
read_meta.name = meta.name;
read_meta.version = 100;
auto reader = codec()->get_segment_meta_reader();
reader->read(dir(), read_meta);
ASSERT_EQ(meta.codec, read_meta.codec); // codec stays nullptr
ASSERT_EQ(meta.name, read_meta.name);
ASSERT_EQ(meta.docs_count, read_meta.docs_count);
ASSERT_EQ(meta.live_docs_count, read_meta.live_docs_count);
ASSERT_EQ(meta.version, read_meta.version);
ASSERT_EQ(meta.size, read_meta.size);
ASSERT_EQ(meta.files, read_meta.files);
ASSERT_EQ(meta.column_store, read_meta.column_store);
}
}
// read segment meta
// read broken meta (live_docs_count > docs_count)
{
irs::segment_meta read_meta;
read_meta.name = meta.name;
read_meta.version = 100;
iresearch::segment_meta meta;
meta.name = "broken_meta_name";
meta.docs_count = 453;
meta.live_docs_count = 1345;
meta.size = 666;
meta.version = 100;
auto reader = codec()->get_segment_meta_reader();
reader->read(dir(), read_meta);
ASSERT_EQ(meta.codec, read_meta.codec); // codec stays nullptr
ASSERT_EQ(meta.name, read_meta.name);
ASSERT_EQ(meta.docs_count, read_meta.docs_count);
ASSERT_EQ(meta.live_docs_count, read_meta.live_docs_count);
ASSERT_EQ(meta.version, read_meta.version);
ASSERT_EQ(meta.files, read_meta.files);
meta.files.emplace("file1");
meta.files.emplace("index_file2");
meta.files.emplace("file3");
meta.files.emplace("stored_file4");
// write segment meta
{
auto writer = codec()->get_segment_meta_writer();
writer->write(dir(), meta);
}
// read segment meta
{
irs::segment_meta read_meta;
read_meta.name = meta.name;
read_meta.version = 100;
auto reader = codec()->get_segment_meta_reader();
ASSERT_THROW(reader->read(dir(), read_meta), irs::index_error);
}
}
}

46
3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp vendored
View File

@ -24,7 +24,6 @@
#include "tests_shared.hpp"
#include "index/index_writer.hpp"
#include "store/memory_directory.hpp"
#include "utils/index_utils.hpp"
NS_LOCAL
@ -48,7 +47,6 @@ TEST(consolidation_test_tier, test_defaults) {
irs::index_utils::consolidate_tier options;
auto policy = irs::index_utils::consolidation_policy(options);
irs::memory_directory dir;
{
irs::index_writer::consolidating_segments_t consolidating_segments;
@ -126,8 +124,6 @@ TEST(consolidation_test_tier, test_defaults) {
}
TEST(consolidation_test_tier, test_skewed_segments) {
irs::memory_directory dir;
{
irs::index_utils::consolidate_tier options;
options.min_segments = 1; // min number of segments per tier to merge at once
@ -421,8 +417,48 @@ TEST(consolidation_test_tier, test_skewed_segments) {
ASSERT_TRUE(candidates.empty());
}
}
// left-skewed distribution
{
const size_t sizes[] = {
9067, 2228, 9023, 0, 9293, 2637, 7529, 291, 4816, 68, 11, 3582, 4298, 4590, 2772, 9021, 32, 1993, 340, 538, 8578, 258, 8731, 5180, 5708, 339, 3830, 1530, 3906, 8714, 3501,
1767, 2695, 458, 286, 2506, 3454, 9191, 9368, 305, 17, 219, 6198, 1562, 6303, 7162, 4601, 2687, 8205, 8321, 4568, 2511, 6629, 9109, 9502, 1412, 357, 5235, 137, 9886, 5607,
1359, 9174, 529, 7074, 8343, 8023, 1618, 6128, 1661, 515, 2388, 2549, 826, 180, 886, 4237, 317, 170, 1532, 1602, 1091, 8953, 1791, 8523, 130, 22, 6319, 6145, 7034, 2006, 52,
9361, 3443, 8228, 1345, 95, 1940, 6432, 609
};
irs::index_meta meta;
for (auto begin = std::begin(sizes), end = std::end(sizes); begin != end; ++begin) {
const size_t i = std::distance(begin, end);
meta.add(irs::segment_meta(std::to_string(i), nullptr, 100, 100, false, irs::segment_meta::file_set{}, *begin));
}
irs::index_utils::consolidate_tier options;
options.min_segments = 10; // min number of segments per tier to merge at once
options.max_segments = 10; // max number of segments per tier to merge at once
options.max_segments_bytes = 250000; // max size of the merge
options.floor_segment_bytes = 50; // smaller segments will be treated as equal to this value
options.lookahead = 0;//irs::integer_traits<size_t>::const_max;
auto policy = irs::index_utils::consolidation_policy(options);
irs::index_writer::consolidating_segments_t consolidating_segments;
std::set<const irs::segment_meta*> candidates;
do {
candidates.clear();
policy(candidates, meta, consolidating_segments);
std::cerr << "Consolidation: ";
for (auto* segment : candidates) {
std::cerr << segment->size << ", ";
}
std::cerr << std::endl;
consolidating_segments.insert(candidates.begin(), candidates.end()); // register candidates for consolidation
} while (!candidates.empty());
}
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

2
3rdParty/iresearch/tests/index/index_tests.cpp vendored
View File

@ -10789,7 +10789,7 @@ class index_test_case_base : public tests::index_test_base {
doc4->indexed.begin(), doc4->indexed.end(),
doc4->stored.begin(), doc4->stored.end()
));
ASSERT_THROW(writer->commit(), irs::io_error);
ASSERT_THROW(writer->commit(), irs::detailed_io_error);
}
// check index, it should be empty

17
arangod/IResearch/IResearchView.cpp
View File

@ -1543,8 +1543,14 @@ int IResearchView::insert(
|| !impl.updateProperties(meta).ok() // update separately since per-instance async jobs already started
|| !impl._metaState.init(properties, error)) {
TRI_set_errno(TRI_ERROR_BAD_PARAMETER);
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failed to initialize arangosearch view from definition, error: " << error;
if (error.empty()) {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failed to initialize arangosearch view '" << impl.name() << "' from definition";
} else {
LOG_TOPIC(WARN, arangodb::iresearch::TOPIC)
<< "failed to initialize arangosearch view '" << impl.name() << "' from definition, error in attribute: " << error;
}
return nullptr;
}
@ -1928,7 +1934,12 @@ arangodb::Result IResearchView::updateProperties(
auto& initialMeta = partialUpdate ? *metaPtr : IResearchViewMeta::DEFAULT();
if (!meta.init(slice, error, initialMeta)) {
return arangodb::Result(TRI_ERROR_BAD_PARAMETER, std::move(error));
return arangodb::Result(
TRI_ERROR_BAD_PARAMETER,
error.empty()
? (std::string("failed to update arangosearch view '") + name() + "' from definition")
: (std::string("failed to update arangosearch view '") + name() + "' from definition, error in attribute: " + error)
);
}
// reset non-updatable values to match current meta

2
arangod/RestHandler/RestViewHandler.cpp
View File

@ -345,7 +345,7 @@ void RestViewHandler::modifyView(bool partialUpdate) {
); // TODO: not force sync?
if (!result.ok()) {
generateError(GeneralResponse::responseCode(result.errorNumber()), result.errorNumber(), result.errorMessage());
generateError(result);
return;
}