Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

issue 511.6: update iresearch to 6e870904eae5eaf891359ed06e7332666c02cb19 (#7819) 

* issue 511.6: update iresearch to 6e870904eae5eaf891359ed06e7332666c02cb19

* backport: add back required MSVC command replacement

* try to address MSVC link issues

* backport: move post-build step into a separate custom target

* another attempt to address cmake build issues
This commit is contained in:
Vasiliy 2018-12-21 15:00:35 +03:00 committed by Andrey Abramov
parent bbd8aa4e0b
commit db07e1ca5e
31 changed files with 1728 additions and 442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
3rdParty/CMakeLists.txt vendored
View File

@ -126,6 +126,9 @@ if (USE_IRESEARCH)
set(ICU_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/V8/v5.7.492.77/third_party/icu")
elseif(MSVC)
add_library(icu-shared STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch shared dependencies
add_library(icudata-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icui18n-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icuuc-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icu-static STATIC IMPORTED GLOBAL)
set_property(TARGET icu-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION_DEBUG "${CMAKE_CURRENT_BINARY_DIR}/build/Debug/lib/icui18n.lib")
@ -139,16 +142,13 @@ if (USE_IRESEARCH)
set(ICU_FOUND TRUE) # ICU built from source in 3rdParty directory
else()
add_library(icu-shared STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch shared dependencies
add_library(icudata-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icui18n-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icuuc-static STATIC IMPORTED GLOBAL) # empty unused library to satisfy IResearch static dependencies
add_library(icu-static STATIC IMPORTED GLOBAL)
set_property(TARGET icu-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION_DEBUG "${ICU_LIBRARY_DIR}/libicui18n.a")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION_RELEASE "${ICU_LIBRARY_DIR}/libicui18n.a")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO "${ICU_LIBRARY_DIR}/libicui18n.a")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION_MINSIZEREL "${ICU_LIBRARY_DIR}/libicui18n.a")
set_property(TARGET icu-static PROPERTY INTERFACE_LINK_LIBRARIES_DEBUG "${ICU_LIBRARY_DIR}/libicuuc.a")
set_property(TARGET icu-static PROPERTY INTERFACE_LINK_LIBRARIES_RELEASE "${ICU_LIBRARY_DIR}/libicuuc.a")
set_property(TARGET icu-static PROPERTY INTERFACE_LINK_LIBRARIES_RELWITHDEBINFO "${ICU_LIBRARY_DIR}/libicuuc.a")
set_property(TARGET icu-static PROPERTY INTERFACE_LINK_LIBRARIES_MINSIZEREL "${ICU_LIBRARY_DIR}/libicuuc.a")
set_property(TARGET icu-static PROPERTY IMPORTED_LOCATION "${ICU_LIBRARY_DIR}/libicui18n.a")
set_property(TARGET icu-static PROPERTY INTERFACE_LINK_LIBRARIES "${ICU_LIBRARY_DIR}/libicuuc.a")
set(ICU_FOUND TRUE) # ICU built from source in 3rdParty directory
endif()
@ -166,12 +166,6 @@ if (USE_IRESEARCH)
set(UNWIND_ROOT "invalid")
set(BFD_ROOT "invalid")
set(IRESEARCH_PREGENERATED_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/iresearch.build/")
set(GENERATED_PARSER_SOURCES "${IRESEARCH_PREGENERATED_ROOT}/core/iql/parser.cc" CACHE PATH "use pregenerated source")
include_directories("${IRESEARCH_PREGENERATED_ROOT}/core/")
include_directories("${IRESEARCH_PREGENERATED_ROOT}/external/snowball/libstemmer/")
set(IRESEARCH_EXCLUDE_STATIC_THIRD_PARTY_LIBS TRUE) # disable linking in of 3rd party libraries automatically
find_package(IResearch REQUIRED) # set IRESEARCH_BUILD_DIR
set(CMAKE_MACOSX_RPATH ON) # suppress cmake warning (use same value as cmake default)

56
3rdParty/iresearch/cmake/FindICU.cmake vendored
View File

@ -131,10 +131,10 @@ if (ICU_INCLUDE_DIR AND ICU_SRC_DIR_UCONV AND ICU_SRC_DIR_CONFIGURE)
)
add_library(icudata-static STATIC IMPORTED GLOBAL)
add_dependencies(icudata-shared icu-build)
set_target_properties(icudata-shared PROPERTIES
add_dependencies(icudata-static icu-build)
set_target_properties(icudata-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_PATH}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icudata${ICU_SHARED_LIBRARY_SUFFIX}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icudata${ICU_STATIC_LIBRARY_SUFFIX}"
)
add_library(icui18n-shared SHARED IMPORTED GLOBAL)
@ -145,10 +145,10 @@ if (ICU_INCLUDE_DIR AND ICU_SRC_DIR_UCONV AND ICU_SRC_DIR_CONFIGURE)
)
add_library(icui18n-static STATIC IMPORTED GLOBAL)
add_dependencies(icui18n-shared icu-build)
set_target_properties(icui18n-shared PROPERTIES
add_dependencies(icui18n-static icu-build)
set_target_properties(icui18n-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_PATH}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icui18n${ICU_SHARED_LIBRARY_SUFFIX}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icui18n${ICU_STATIC_LIBRARY_SUFFIX}"
)
add_library(icuuc-shared SHARED IMPORTED GLOBAL)
@ -159,10 +159,10 @@ if (ICU_INCLUDE_DIR AND ICU_SRC_DIR_UCONV AND ICU_SRC_DIR_CONFIGURE)
)
add_library(icuuc-static STATIC IMPORTED GLOBAL)
add_dependencies(icuuc-shared icu-build)
set_target_properties(icuuc-shared PROPERTIES
add_dependencies(icuuc-static icu-build)
set_target_properties(icuuc-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_PATH}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icuuc${ICU_SHARED_LIBRARY_SUFFIX}"
IMPORTED_LOCATION "${ICU_LIBRARY_PATH}/${ICU_STATIC_LIBRARY_PREFIX}icuuc${ICU_STATIC_LIBRARY_SUFFIX}"
)
add_library(icu-shared SHARED IMPORTED GLOBAL)
@ -295,7 +295,44 @@ if (ICU_INCLUDE_DIR
list(APPEND ICU_STATIC_LIBS ${ICU_STATIC_LIBRARY_DT} ${ICU_STATIC_LIBRARY_IN} ${ICU_STATIC_LIBRARY_UC})
endif()
add_library(icudata-shared SHARED IMPORTED GLOBAL)
set_target_properties(icudata-shared PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_SHARED_LIBRARY_DT}"
)
add_library(icudata-static STATIC IMPORTED GLOBAL)
set_target_properties(icudata-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_STATIC_LIBRARY_DT}"
)
add_library(icui18n-shared SHARED IMPORTED GLOBAL)
set_target_properties(icui18n-shared PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_SHARED_LIBRARY_IN}"
)
add_library(icui18n-static STATIC IMPORTED GLOBAL)
set_target_properties(icui18n-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_STATIC_LIBRARY_IN}"
)
add_library(icuuc-shared SHARED IMPORTED GLOBAL)
set_target_properties(icuuc-shared PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_SHARED_LIBRARY_UC}"
)
add_library(icuuc-static STATIC IMPORTED GLOBAL)
set_target_properties(icuuc-static PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}"
IMPORTED_LOCATION "${ICU_STATIC_LIBRARY_UC}"
)
add_library(icu-shared SHARED IMPORTED GLOBAL)
add_dependencies(icu-shared icudata-shared icui18n-shared icuuc-shared)
set_property(TARGET icu-shared PROPERTY INTERFACE_INCLUDE_DIRECTORIES # $<TARGET_PROPERTY:tgt,INCLUDE_DIRECTORIES> does not get expanded
"${ICU_INCLUDE_DIR}"
)
@ -308,6 +345,7 @@ if (ICU_INCLUDE_DIR
)
add_library(icu-static STATIC IMPORTED GLOBAL)
add_dependencies(icu-static icudata-static icui18n-static icuuc-static)
set_property(TARGET icu-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES # $<TARGET_PROPERTY:tgt,INCLUDE_DIRECTORIES> does not get expanded
"${ICU_INCLUDE_DIR}"
)

152
3rdParty/iresearch/core/CMakeLists.txt vendored
View File

@ -552,9 +552,7 @@ if(MSVC)
)
endif()
if (IRESEARCH_EXCLUDE_STATIC_THIRD_PARTY_LIBS)
#NOOP, do not link in 3rd party libraries
elseif (MSVC)
if (MSVC)
unset(IResearch_STATIC_LIBRARIES)
foreach(ELEMENT
@ -562,7 +560,9 @@ elseif (MSVC)
"$<$<CONFIG:Debug>:${Boost_STATIC_sharedRT_LOCALE_LIBRARY_DEBUG}>$<$<NOT:$<CONFIG:Debug>>:${Boost_STATIC_sharedRT_LOCALE_LIBRARY_RELEASE}>"
"$<$<CONFIG:Debug>:${Boost_STATIC_sharedRT_SYSTEM_LIBRARY_DEBUG}>$<$<NOT:$<CONFIG:Debug>>:${Boost_STATIC_sharedRT_SYSTEM_LIBRARY_RELEASE}>"
${Unwind_STATIC_LIBS}
$<TARGET_FILE:icu-static>
"$<TARGET_FILE:icudata-static>" # must expand icu-static into components
"$<TARGET_FILE:icui18n-static>" # must expand icu-static into components
"$<TARGET_FILE:icuuc-static>" # must expand icu-static into components
"$<TARGET_FILE:lz4_static>"
"$<TARGET_FILE:stemmer-static>"
"$<TARGET_FILE:${IResearch_TARGET_NAME}-analyzer-delimited-static>"
@ -574,11 +574,39 @@ elseif (MSVC)
set(IResearch_STATIC_LIBRARIES "${IResearch_STATIC_LIBRARIES} \"${ELEMENT}\"")
endforeach()
set(IResearch_STATIC_LIBRARIES_CMD "lib.exe \"/OUT:$<TARGET_FILE:${IResearch_TARGET_NAME}-static>\" \"$<TARGET_FILE:${IResearch_TARGET_NAME}-static>\" ${IResearch_STATIC_LIBRARIES}")
string(REPLACE " " ";" IResearch_STATIC_LIBRARIES_CMD "${IResearch_STATIC_LIBRARIES_CMD}") # COMMAND requires ';' instead of ' '
set(IResearch_STATIC_LIBRARIES_OUT "${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libiresearch-sa${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(IResearch_STATIC_LIBRARIES_CMD "lib.exe \"/OUT:${IResearch_STATIC_LIBRARIES_OUT}\" \"$<TARGET_FILE:${IResearch_TARGET_NAME}-static>\" ${IResearch_STATIC_LIBRARIES}")
add_custom_command(
TARGET ${IResearch_TARGET_NAME}-static POST_BUILD
OUTPUT "${IResearch_STATIC_LIBRARIES_OUT}"
DEPENDS ${IResearch_TARGET_NAME}-static
COMMAND ${IResearch_STATIC_LIBRARIES_CMD}
VERBATIM
)
add_library(${IResearch_TARGET_NAME}-static-allinone
STATIC IMPORTED GLOBAL
)
add_custom_target(${IResearch_TARGET_NAME}-static-allinone-build
DEPENDS "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_target_properties(${IResearch_TARGET_NAME}-static-allinone
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "$<TARGET_PROPERTY:${IResearch_TARGET_NAME}-static,INTERFACE_INCLUDE_DIRECTORIES>"
IMPORTED_LOCATION "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_property(TARGET ${IResearch_TARGET_NAME}-static-allinone
PROPERTY
INTERFACE_LINK_LIBRARIES # additional IMPORTED_LOCATION value list (non-static libraries)
${ATOMIC_LIBRARY} # non-static library
${DL_LIBRARY} # non-static library
${MSVC_ONLY_LIBRARIES} # non-static library
)
add_dependencies(${IResearch_TARGET_NAME}-static-allinone
${IResearch_TARGET_NAME}-static-allinone-build
)
unset(IResearch_STATIC_SCRT_LIBRARIES)
@ -588,7 +616,9 @@ elseif (MSVC)
"$<$<CONFIG:Debug>:${Boost_STATIC_staticRT_LOCALE_LIBRARY_DEBUG}>$<$<NOT:$<CONFIG:Debug>>:${Boost_STATIC_staticRT_LOCALE_LIBRARY_RELEASE}>"
"$<$<CONFIG:Debug>:${Boost_STATIC_staticRT_SYSTEM_LIBRARY_DEBUG}>$<$<NOT:$<CONFIG:Debug>>:${Boost_STATIC_staticRT_SYSTEM_LIBRARY_RELEASE}>"
${Unwind_STATIC_LIBS}
$<TARGET_FILE:icu-static>
"$<TARGET_FILE:icudata-static>" # must expand icu-static into components
"$<TARGET_FILE:icui18n-static>" # must expand icu-static into components
"$<TARGET_FILE:icuuc-static>" # must expand icu-static into components
"$<TARGET_FILE:lz4_static>"
"$<TARGET_FILE:stemmer-static>"
"$<TARGET_FILE:${IResearch_TARGET_NAME}-analyzer-delimited-static-scrt>"
@ -600,11 +630,39 @@ elseif (MSVC)
set(IResearch_STATIC_SCRT_LIBRARIES "${IResearch_STATIC_SCRT_LIBRARIES} \"${ELEMENT}\"")
endforeach()
set(IResearch_STATIC_SCRT_LIBRARIES_CMD "lib.exe \"/OUT:$<TARGET_FILE:${IResearch_TARGET_NAME}-static-scrt>\" \"$<TARGET_FILE:${IResearch_TARGET_NAME}-static-scrt>\" ${IResearch_STATIC_SCRT_LIBRARIES}")
string(REPLACE " " ";" IResearch_STATIC_SCRT_LIBRARIES_CMD "${IResearch_STATIC_SCRT_LIBRARIES_CMD}") # COMMAND requires ';' instead of ' '
set(IResearch_STATIC_SCRT_LIBRARIES_OUT "${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libiresearch-srct-sa${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(IResearch_STATIC_SCRT_LIBRARIES_CMD "lib.exe \"/OUT:${IResearch_STATIC_SCRT_LIBRARIES_OUT}\" \"$<TARGET_FILE:${IResearch_TARGET_NAME}-static-scrt>\" ${IResearch_STATIC_SCRT_LIBRARIES}")
add_custom_command(
TARGET ${IResearch_TARGET_NAME}-static-scrt POST_BUILD
OUTPUT "${IResearch_STATIC_SCRT_LIBRARIES_OUT}"
DEPENDS ${IResearch_TARGET_NAME}-static-scrt
COMMAND ${IResearch_STATIC_SCRT_LIBRARIES_CMD}
VERBATIM
)
add_library(${IResearch_TARGET_NAME}-static-scrt-allinone
STATIC IMPORTED GLOBAL
)
add_custom_target(${IResearch_TARGET_NAME}-static-scrt-allinone-build
DEPENDS "${IResearch_STATIC_SCRT_LIBRARIES_OUT}"
)
set_target_properties(${IResearch_TARGET_NAME}-static-scrt-allinone
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "$<TARGET_PROPERTY:${IResearch_TARGET_NAME}-static-srct,INTERFACE_INCLUDE_DIRECTORIES>"
IMPORTED_LOCATION "${IResearch_STATIC_SCRT_LIBRARIES_OUT}"
)
set_property(TARGET ${IResearch_TARGET_NAME}-static-scrt-allinone
PROPERTY
INTERFACE_LINK_LIBRARIES # additional IMPORTED_LOCATION value list (non-static libraries)
${ATOMIC_LIBRARY} # non-static library
${DL_LIBRARY} # non-static library
${MSVC_ONLY_LIBRARIES} # non-static library
)
add_dependencies(${IResearch_TARGET_NAME}-static-scrt-allinone
${IResearch_TARGET_NAME}-static-scrt-allinone-build
)
elseif (APPLE)
unset(IResearch_STATIC_LIBRARIES)
@ -615,7 +673,9 @@ elseif (APPLE)
${Boost_STATIC_sharedRT_SYSTEM_LIBRARY_DEBUG}
${Boost_STATIC_sharedRT_THREAD_LIBRARY_DEBUG}
${Unwind_STATIC_LIBS}
$<TARGET_FILE:icu-static>
"$<TARGET_FILE:icudata-static>" # must expand icu-static into components
"$<TARGET_FILE:icui18n-static>" # must expand icu-static into components
"$<TARGET_FILE:icuuc-static>" # must expand icu-static into components
"$<TARGET_FILE:lz4_static>"
"$<TARGET_FILE:stemmer-static>"
"$<TARGET_FILE:${IResearch_TARGET_NAME}-analyzer-delimited-static>"
@ -627,12 +687,40 @@ elseif (APPLE)
set(IResearch_STATIC_LIBRARIES "${IResearch_STATIC_LIBRARIES} '${ELEMENT}'")
endforeach ()
set(IResearch_STATIC_LIBRARIES_CMD "mv '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>' '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'.base && libtool -static -o '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>' -a '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'.base ${IResearch_STATIC_LIBRARIES} || rm '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'")
set(IResearch_STATIC_LIBRARIES_OUT "${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libiresearch-sa${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(IResearch_STATIC_LIBRARIES_CMD "libtool -static -o '${IResearch_STATIC_LIBRARIES_OUT}' -a '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>' ${IResearch_STATIC_LIBRARIES}")
add_custom_command(
TARGET ${IResearch_TARGET_NAME}-static POST_BUILD
OUTPUT "${IResearch_STATIC_LIBRARIES_OUT}"
DEPENDS ${IResearch_TARGET_NAME}-static
COMMAND sh -c ${IResearch_STATIC_LIBRARIES_CMD}
VERBATIM
)
add_library(${IResearch_TARGET_NAME}-static-allinone
STATIC IMPORTED GLOBAL
)
add_custom_target(${IResearch_TARGET_NAME}-static-allinone-build
DEPENDS "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_target_properties(${IResearch_TARGET_NAME}-static-allinone
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "$<TARGET_PROPERTY:${IResearch_TARGET_NAME}-static,INTERFACE_INCLUDE_DIRECTORIES>"
IMPORTED_LOCATION "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_property(TARGET ${IResearch_TARGET_NAME}-static-allinone
PROPERTY
INTERFACE_LINK_LIBRARIES # additional IMPORTED_LOCATION value list (non-static libraries)
${ATOMIC_LIBRARY} # non-static library
${DL_LIBRARY} # non-static library
${MSVC_ONLY_LIBRARIES} # non-static library
)
add_dependencies(${IResearch_TARGET_NAME}-static-allinone
${IResearch_TARGET_NAME}-static-allinone-build
)
else()
unset(IResearch_STATIC_LIBRARIES)
@ -642,7 +730,9 @@ else()
${Boost_STATIC_sharedRT_SYSTEM_LIBRARY_DEBUG}
${Boost_STATIC_sharedRT_THREAD_LIBRARY_DEBUG}
${Unwind_STATIC_LIBS}
$<TARGET_FILE:icu-static>
"$<TARGET_FILE:icudata-static>" # must expand icu-static into components
"$<TARGET_FILE:icui18n-static>" # must expand icu-static into components
"$<TARGET_FILE:icuuc-static>" # must expand icu-static into components
"$<TARGET_FILE:lz4_static>"
"$<TARGET_FILE:stemmer-static>"
"$<TARGET_FILE:${IResearch_TARGET_NAME}-analyzer-delimited-static>"
@ -654,12 +744,40 @@ else()
set(IResearch_STATIC_LIBRARIES "${IResearch_STATIC_LIBRARIES} addlib '${ELEMENT}'\\n")
endforeach ()
set(IResearch_STATIC_LIBRARIES_CMD "`which echo` -e \"create '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'\\n addlib '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'\\n ${IResearch_STATIC_LIBRARIES} save\\n end\" | ar -M")
set(IResearch_STATIC_LIBRARIES_OUT "${CMAKE_BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libresearch-sa${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(IResearch_STATIC_LIBRARIES_CMD "`which echo` -e \"create '${IResearch_STATIC_LIBRARIES_OUT}'\\n addlib '$<TARGET_FILE:${IResearch_TARGET_NAME}-static>'\\n ${IResearch_STATIC_LIBRARIES} save\\n end\" | ar -M")
add_custom_command(
TARGET ${IResearch_TARGET_NAME}-static POST_BUILD
OUTPUT "${IResearch_STATIC_LIBRARIES_OUT}"
DEPENDS ${IResearch_TARGET_NAME}-static
COMMAND sh -c ${IResearch_STATIC_LIBRARIES_CMD}
VERBATIM
)
add_library(${IResearch_TARGET_NAME}-static-allinone
STATIC IMPORTED GLOBAL
)
add_custom_target(${IResearch_TARGET_NAME}-static-allinone-build
DEPENDS "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_target_properties(${IResearch_TARGET_NAME}-static-allinone
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "$<TARGET_PROPERTY:${IResearch_TARGET_NAME}-static,INTERFACE_INCLUDE_DIRECTORIES>"
IMPORTED_LOCATION "${IResearch_STATIC_LIBRARIES_OUT}"
)
set_property(TARGET ${IResearch_TARGET_NAME}-static-allinone
PROPERTY
INTERFACE_LINK_LIBRARIES # additional IMPORTED_LOCATION value list (non-static libraries)
${ATOMIC_LIBRARY} # non-static library
${DL_LIBRARY} # non-static library
${MSVC_ONLY_LIBRARIES} # non-static library
)
add_dependencies(${IResearch_TARGET_NAME}-static-allinone
${IResearch_TARGET_NAME}-static-allinone-build
)
endif()
################################################################################

23
3rdParty/iresearch/core/formats/formats_10.cpp vendored
View File

@ -993,9 +993,7 @@ FORCE_INLINE void skip_offsets(index_input& in) {
///////////////////////////////////////////////////////////////////////////////
class doc_iterator : public irs::doc_iterator {
public:
DECLARE_UNIQUE_PTR(doc_iterator);
DEFINE_FACTORY_INLINE(doc_iterator);
DECLARE_SHARED_PTR(doc_iterator);
doc_iterator() NOEXCEPT
: skip_levels_(1),
@ -1047,6 +1045,8 @@ class doc_iterator : public irs::doc_iterator {
}
seek_to_block(target);
// FIXME binary search instead of linear
irs::seek(*this, target);
return value();
}
@ -1458,7 +1458,7 @@ class pos_iterator: public position {
}
if (count < left) {
buf_pos_ += uint32_t(count);
buf_pos_ += count;
}
clear();
value_ = 0;
@ -5087,7 +5087,8 @@ void postings_reader::decode(
irs::doc_iterator::ptr postings_reader::iterator(
const flags& field,
const attribute_view& attrs,
const flags& req) {
const flags& req
) {
// compile field features
const auto features = ::features(field);
// get enabled features:
@ -5099,19 +5100,19 @@ irs::doc_iterator::ptr postings_reader::iterator(
// 'operator|' in the following switch statement
switch (enabled) {
case features::FREQ_POS_OFFS_PAY:
it = doc_iterator::make<pos_doc_iterator<offs_pay_iterator>>();
it = std::make_shared<pos_doc_iterator<offs_pay_iterator>>();
break;
case features::FREQ_POS_OFFS:
it = doc_iterator::make<pos_doc_iterator<offs_iterator>>();
it = std::make_shared<pos_doc_iterator<offs_iterator>>();
break;
case features::FREQ_POS_PAY:
it = doc_iterator::make<pos_doc_iterator<pay_iterator>>();
it = std::make_shared<pos_doc_iterator<pay_iterator>>();
break;
case features::FREQ_POS:
it = doc_iterator::make<pos_doc_iterator<pos_iterator>>();
it = std::make_shared<pos_doc_iterator<pos_iterator>>();
break;
default:
it = doc_iterator::make<doc_iterator>();
it = std::make_shared<doc_iterator>();
}
it->prepare(
@ -5119,7 +5120,7 @@ irs::doc_iterator::ptr postings_reader::iterator(
doc_in_.get(), pos_in_.get(), pay_in_.get()
);
return IMPLICIT_MOVE_WORKAROUND(it);
return it;
}
#if defined(_MSC_VER)

62
3rdParty/iresearch/core/index/index_writer.cpp vendored
View File

@ -599,10 +599,10 @@ index_writer::documents_context::document::~document() NOEXCEPT {
}
index_writer::documents_context::~documents_context() NOEXCEPT {
// FIXME TODO move emplace into active_segment_context destructor
assert(segment_.ctx().use_count() == segment_use_count_); // failure may indicate a dangling 'document' instance
try {
// FIXME TODO move emplace into active_segment_context destructor
writer_.get_flush_context()->emplace(std::move(segment_)); // commit segment
} catch (...) {
reset(); // abort segment
@ -708,15 +708,15 @@ index_writer::flush_context_ptr index_writer::documents_context::update_segment(
assert(segment_.ctx());
assert(segment_.ctx()->writer_);
auto& segment = *(segment_.ctx());
const auto& limits = writer_.segment_limits_;
auto& writer = *segment.writer_;
if (writer.initialized()) {
auto segment_docs_max = writer_.segment_limits_.segment_docs_max.load();
auto segment_memory_max = writer_.segment_limits_.segment_memory_max.load();
// if not reached the limit of the current segment then use it
if ((!limits.segment_docs_max
|| limits.segment_docs_max > writer.docs_cached()) // too many docs
&& (!limits.segment_memory_max
|| limits.segment_memory_max > writer.memory_active()) // too much memory
if ((!segment_docs_max || segment_docs_max > writer.docs_cached()) // too many docs
&& (!segment_memory_max || segment_memory_max > writer.memory_active()) // too much memory
&& !doc_limits::eof(writer.docs_cached())) { // segment full
return ctx;
}
@ -724,7 +724,7 @@ index_writer::flush_context_ptr index_writer::documents_context::update_segment(
// force a flush of a full segment
IR_FRMT_TRACE(
"Flushing segment '%s', docs=" IR_SIZE_T_SPECIFIER ", memory=" IR_SIZE_T_SPECIFIER ", docs limit=" IR_SIZE_T_SPECIFIER ", memory limit=" IR_SIZE_T_SPECIFIER "",
writer.name().c_str(), writer.docs_cached(), writer.memory_active(), limits.segment_docs_max, limits.segment_memory_max
writer.name().c_str(), writer.docs_cached(), writer.memory_active(), segment_docs_max, segment_memory_max
);
try {
@ -864,6 +864,11 @@ void index_writer::flush_context::emplace(active_segment_context&& segment) {
if (!ctx.dirty_) {
assert(freelist_node);
assert(segment.ctx_.use_count() == 2); // +1 for 'active_segment_context::ctx_', +1 for 'pending_segment_context::segment_'
auto& segments_active = *(segment.segments_active_);
auto segments_active_decrement =
irs::make_finally([&segments_active]()->void { --segments_active; }); // release hold (delcare before aquisition since operator++() is noexcept)
++segments_active; // increment counter to hold reservation while segment_context is being released and added to the freelist
segment = active_segment_context(); // reset before adding to freelist to garantee proper use_count() in get_segment_context(...)
pending_segment_contexts_freelist_.push(*freelist_node); // add segment_context to free-list
}
@ -1054,7 +1059,7 @@ index_writer::index_writer(
directory& dir,
format::ptr codec,
size_t segment_pool_size,
const segment_limits& segment_limits,
const segment_options& segment_limits,
index_meta&& meta,
committed_state_t&& committed_state
) NOEXCEPT:
@ -1142,7 +1147,7 @@ index_writer::ptr index_writer::make(
directory& dir,
format::ptr codec,
OpenMode mode,
const options& opts /*= options()*/
const init_options& opts /*= init_options()*/
) {
std::vector<index_file_refs::ref_t> file_refs;
index_lock::ptr lock;
@ -1203,7 +1208,7 @@ index_writer::ptr index_writer::make(
dir,
codec,
opts.segment_pool_size,
segment_limits(opts),
segment_options(opts),
std::move(meta),
std::move(comitted_state)
);
@ -1214,6 +1219,10 @@ index_writer::ptr index_writer::make(
return writer;
}
void index_writer::options(const segment_options& opts) {
segment_limits_ = opts;
}
index_writer::~index_writer() NOEXCEPT {
assert(!segments_active_.load()); // failure may indicate a dangling 'document' instance
cached_readers_.clear();
@ -1606,6 +1615,20 @@ index_writer::flush_context_ptr index_writer::get_flush_context(bool shared /*=
index_writer::active_segment_context index_writer::get_segment_context(
flush_context& ctx
) {
auto segments_active_decrement =
irs::make_finally([this]()->void { --segments_active_; }); // release reservation (delcare before aquisition since operator++() is noexcept)
auto segments_active = ++segments_active_; // increment counter to aquire reservation, if another thread tries to reserve last context then it'll be over limit
auto segment_count_max = segment_limits_.segment_count_max.load();
// no free segment_context available and maximum number of segments reached
// must return to caller so as to unlock/relock flush_context before retrying
// to get a new segment so as to avoid a deadlock due to a read-write-read
// situation for flush_context::flush_mutex_ with threads trying to lock
// flush_context::flush_mutex_ to return their segment_context
if (segment_count_max && segment_count_max < segments_active) { // '<' to account for +1 reservation
return active_segment_context();
}
auto* freelist_node = static_cast<flush_context::pending_segment_context*>(
ctx.pending_segment_contexts_freelist_.pop()
); // only nodes of type 'pending_segment_context' are added to 'pending_segment_contexts_freelist_'
@ -1619,16 +1642,6 @@ index_writer::active_segment_context index_writer::get_segment_context(
);
}
// no free segment_context available and maximum number of segments reached
// must return to caller so as to unlock/relock flush_context before retrying
// to get a new segment so as to avoid a deadlock due to a read-write-read
// situation for flush_context::flush_mutex_ with threads trying to lock
// flush_context::flush_mutex_ to return their segment_context
if (segment_limits_.segment_count_max
&& segments_active_.load() >= segment_limits_.segment_count_max) {
return active_segment_context();
}
// ...........................................................................
// should allocate a new segment_context from the pool
// ...........................................................................
@ -1638,6 +1651,13 @@ index_writer::active_segment_context index_writer::get_segment_context(
};
auto segment_ctx =
segment_writer_pool_.emplace(dir_, std::move(meta_generator)).release();
auto segment_memory_max = segment_limits_.segment_memory_max.load();
// recreate writer if it reserved more memory than allowed by current limits
if (segment_memory_max &&
segment_memory_max < segment_ctx->writer_->memory_reserved()) {
segment_ctx->writer_ = segment_writer::make(segment_ctx->dir_);
}
return active_segment_context(segment_ctx, segments_active_);
}
@ -2202,4 +2222,4 @@ NS_END
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

81
3rdParty/iresearch/core/index/index_writer.hpp vendored
View File

@ -132,7 +132,7 @@ class IRESEARCH_API index_writer:
private:
friend struct flush_context; // for flush_context::emplace(...)
segment_context_ptr ctx_{};
segment_context_ptr ctx_{nullptr};
flush_context* flush_ctx_{nullptr}; // nullptr will not match any flush_context
size_t pending_segment_context_offset_; // segment offset in flush_ctx_->pending_segment_contexts_
std::atomic<size_t>* segments_active_; // reference to index_writer::segments_active_
@ -375,21 +375,9 @@ class IRESEARCH_API index_writer:
};
//////////////////////////////////////////////////////////////////////////////
/// @brief options the the writer should use after creation
/// @brief options the the writer should use for segments
//////////////////////////////////////////////////////////////////////////////
struct options {
////////////////////////////////////////////////////////////////////////////
/// @brief aquire an exclusive lock on the repository to guard against index
/// corruption from multiple index_writers
////////////////////////////////////////////////////////////////////////////
bool lock_repository{true};
////////////////////////////////////////////////////////////////////////////
/// @brief number of memory blocks to cache by the internal memory pool
/// 0 == use default from memory_allocator::global()
////////////////////////////////////////////////////////////////////////////
size_t memory_pool_size{0};
struct segment_options {
////////////////////////////////////////////////////////////////////////////
/// @brief segment aquisition requests will block and wait for free segments
/// after this many segments have been aquired e.g. via documents()
@ -412,6 +400,23 @@ class IRESEARCH_API index_writer:
/// 0 == unlimited
////////////////////////////////////////////////////////////////////////////
size_t segment_memory_max{0};
};
//////////////////////////////////////////////////////////////////////////////
/// @brief options the the writer should use after creation
//////////////////////////////////////////////////////////////////////////////
struct init_options: public segment_options {
////////////////////////////////////////////////////////////////////////////
/// @brief aquire an exclusive lock on the repository to guard against index
/// corruption from multiple index_writers
////////////////////////////////////////////////////////////////////////////
bool lock_repository{true};
////////////////////////////////////////////////////////////////////////////
/// @brief number of memory blocks to cache by the internal memory pool
/// 0 == use default from memory_allocator::global()
////////////////////////////////////////////////////////////////////////////
size_t memory_pool_size{0};
////////////////////////////////////////////////////////////////////////////
/// @brief number of free segments cached in the segment pool for reuse
@ -419,7 +424,7 @@ class IRESEARCH_API index_writer:
////////////////////////////////////////////////////////////////////////////
size_t segment_pool_size{128}; // arbitrary size
options() {}; // GCC5 requires non-default definition
init_options() {}; // GCC5 requires non-default definition
};
struct segment_hash {
@ -439,21 +444,6 @@ class IRESEARCH_API index_writer:
}
}; // segment_equal
//////////////////////////////////////////////////////////////////////////////
/// @brief limits the the writer should use for segments
//////////////////////////////////////////////////////////////////////////////
// FIXME TODO inherit options from segment_limits
struct segment_limits {
size_t segment_count_max; // @see options::max_segment_count
size_t segment_docs_max; // @see options::max_segment_docs
size_t segment_memory_max; // @see options::max_segment_memory
segment_limits(const options& opts) NOEXCEPT
: segment_count_max(opts.segment_count_max),
segment_docs_max(opts.segment_docs_max),
segment_memory_max(opts.segment_memory_max) {
}
};
// works faster than std::unordered_set<string_ref>
typedef std::unordered_set<
const segment_meta*,
@ -556,9 +546,15 @@ class IRESEARCH_API index_writer:
directory& dir,
format::ptr codec,
OpenMode mode,
const options& opts = options()
const init_options& opts = init_options()
);
////////////////////////////////////////////////////////////////////////////
/// @brief modify the runtime segment options as per the specified values
/// options will apply no later than after the next commit()
////////////////////////////////////////////////////////////////////////////
void options(const segment_options& opts);
////////////////////////////////////////////////////////////////////////////
/// @brief begins the two-phase transaction
/// @returns true if transaction has been sucessflully started
@ -795,6 +791,23 @@ class IRESEARCH_API index_writer:
void reset() NOEXCEPT;
};
struct segment_limits {
std::atomic<size_t> segment_count_max; // @see segment_options::max_segment_count
std::atomic<size_t> segment_docs_max; // @see segment_options::max_segment_docs
std::atomic<size_t> segment_memory_max; // @see segment_options::max_segment_memory
segment_limits(const segment_options& opts) NOEXCEPT
: segment_count_max(opts.segment_count_max),
segment_docs_max(opts.segment_docs_max),
segment_memory_max(opts.segment_memory_max) {
}
segment_limits& operator=(const segment_options& opts) NOEXCEPT {
segment_count_max.store(opts.segment_count_max);
segment_docs_max.store(opts.segment_docs_max);
segment_memory_max.store(opts.segment_memory_max);
return *this;
}
};
typedef std::shared_ptr<
std::pair<std::shared_ptr<index_meta>,
file_refs_t
@ -953,7 +966,7 @@ class IRESEARCH_API index_writer:
directory& dir,
format::ptr codec,
size_t segment_pool_size,
const segment_limits& segment_limits,
const segment_options& segment_limits,
index_meta&& meta,
committed_state_t&& committed_state
) NOEXCEPT;
@ -990,4 +1003,4 @@ class IRESEARCH_API index_writer:
NS_END
#endif
#endif

47
3rdParty/iresearch/core/search/bm25.cpp vendored
View File

@ -199,28 +199,28 @@ struct stats final : stored_attribute {
stats() = default;
virtual void clear() {
idf = 1.f;
idf = 0.f;
norm_const = 1.f;
norm_length = 0.f;
}
float_t idf; // precomputed idf value
float_t norm_const; // precomputed k*(1-b)
float_t norm_length; // precomputed k*b/avgD
float_t idf{ 0.f }; // precomputed idf value
float_t norm_const{ 1.f }; // precomputed k*(1-b)
float_t norm_length{ 0.f }; // precomputed k*b/avgD
}; // stats
DEFINE_ATTRIBUTE_TYPE(iresearch::bm25::stats);
DEFINE_ATTRIBUTE_TYPE(irs::bm25::stats);
DEFINE_FACTORY_DEFAULT(stats);
typedef bm25_sort::score_t score_t;
class scorer : public iresearch::sort::scorer_base<bm25::score_t> {
class scorer : public irs::sort::scorer_base<bm25::score_t> {
public:
DEFINE_FACTORY_INLINE(scorer);
scorer(
float_t k,
iresearch::boost::boost_t boost,
irs::boost::boost_t boost,
const bm25::stats* stats,
const frequency* freq) NOEXCEPT
: freq_(freq ? freq : &EMPTY_FREQ),
@ -237,7 +237,7 @@ class scorer : public iresearch::sort::scorer_base<bm25::score_t> {
protected:
FORCE_INLINE float_t tf() const NOEXCEPT {
return float_t(std::sqrt(freq_->value));
};
}
const frequency* freq_; // document frequency
float_t num_; // partially precomputed numerator : boost * (k + 1) * idf
@ -250,10 +250,10 @@ class norm_scorer final : public scorer {
norm_scorer(
float_t k,
iresearch::boost::boost_t boost,
irs::boost::boost_t boost,
const bm25::stats* stats,
const frequency* freq,
const iresearch::norm* norm) NOEXCEPT
const irs::norm* norm) NOEXCEPT
: scorer(k, boost, stats, freq),
norm_(norm) {
assert(norm_);
@ -271,11 +271,11 @@ class norm_scorer final : public scorer {
}
private:
const iresearch::norm* norm_;
const irs::norm* norm_;
float_t norm_length_{ 0.f }; // precomputed 'k*b/avgD' if norms presetn, '0' otherwise
}; // norm_scorer
class collector final : public iresearch::sort::collector {
class collector final : public irs::sort::collector {
public:
collector(float_t k, float_t b) NOEXCEPT
: k_(k), b_(b) {
@ -304,15 +304,17 @@ class collector final : public iresearch::sort::collector {
attribute_store& filter_attrs,
const index_reader& /*index*/
) override {
auto& bm25stats = filter_attrs.emplace<stats>();
bool is_new; // stats weren't already initialized
auto& bm25stats = filter_attrs.try_emplace<stats>(is_new);
// precomputed idf value
bm25stats->idf =
bm25stats->idf +=
float_t(std::log(1 + ((docs_with_field - docs_with_term + 0.5)/(docs_with_term + 0.5))));
assert(bm25stats->idf >= 0);
if (b_ == 0.f) {
// BM11 without norms
// - stats were already initialized
// - BM15 without norms
if (!is_new || b_ == 0.f) {
return;
}
@ -337,7 +339,7 @@ class collector final : public iresearch::sort::collector {
float_t b_;
}; // collector
class sort final : iresearch::sort::prepared_basic<bm25::score_t> {
class sort final : irs::sort::prepared_basic<bm25::score_t> {
public:
DEFINE_FACTORY_INLINE(prepared);
@ -354,7 +356,7 @@ class sort final : iresearch::sort::prepared_basic<bm25::score_t> {
return FEATURES[b_ != 0.f];
}
virtual iresearch::sort::collector::ptr prepare_collector() const override {
virtual irs::sort::collector::ptr prepare_collector() const override {
return irs::sort::collector::make<bm25::collector>(k_, b_);
}
@ -369,7 +371,7 @@ class sort final : iresearch::sort::prepared_basic<bm25::score_t> {
}
if (b_ != 0.f) {
auto& norm = query_attrs.get<iresearch::norm>();
auto& norm = query_attrs.get<irs::norm>();
if (norm && norm->reset(segment, field.meta().norm, *doc_attrs.get<document>())) {
return bm25::scorer::make<bm25::norm_scorer>(
@ -398,13 +400,16 @@ class sort final : iresearch::sort::prepared_basic<bm25::score_t> {
NS_END // bm25
DEFINE_SORT_TYPE_NAMED(iresearch::bm25_sort, "bm25");
DEFINE_SORT_TYPE_NAMED(irs::bm25_sort, "bm25");
DEFINE_FACTORY_DEFAULT(irs::bm25_sort);
bm25_sort::bm25_sort(
float_t k /*= 1.2f*/,
float_t b /*= 0.75f*/
): sort(bm25_sort::type()), k_(k), b_(b) {
) NOEXCEPT
: sort(bm25_sort::type()),
k_(k),
b_(b) {
}
/*static*/ void bm25_sort::init() {

10
3rdParty/iresearch/core/search/bm25.hpp vendored
View File

@ -47,13 +47,13 @@ class bm25_sort : public sort {
typedef float_t score_t;
bm25_sort(float_t k = K(), float_t b = B());
explicit bm25_sort(float_t k = K(), float_t b = B()) NOEXCEPT;
float_t k() const { return k_; }
void k(float_t k) { k_ = k; }
float_t k() const NOEXCEPT { return k_; }
void k(float_t k) NOEXCEPT { k_ = k; }
float_t b() const { return b_; }
void b(float_t b) { b_ = b; }
float_t b() const NOEXCEPT { return b_; }
void b(float_t b) NOEXCEPT { b_ = b; }
// returns 'true' if current scorer is 'bm11'
bool bm11() const NOEXCEPT {

2
3rdParty/iresearch/core/search/filter.cpp vendored
View File

@ -63,7 +63,7 @@ filter::prepared::ptr filter::prepared::empty() {
);
}
filter::prepared::prepared(attribute_store&& attrs)
filter::prepared::prepared(attribute_store&& attrs) NOEXCEPT
: attrs_(std::move(attrs)) {
}

2
3rdParty/iresearch/core/search/filter.hpp vendored
View File

@ -93,7 +93,7 @@ class IRESEARCH_API filter {
static prepared::ptr empty();
prepared() = default;
explicit prepared(attribute_store&& attrs);
explicit prepared(attribute_store&& attrs) NOEXCEPT;
virtual ~prepared();
using util::attribute_store_provider::attributes;

80
3rdParty/iresearch/core/search/phrase_filter.cpp vendored
View File

@ -81,17 +81,18 @@ struct phrase_state {
class phrase_query : public filter::prepared {
public:
typedef states_cache<phrase_state> states_t;
typedef std::pair<
attribute_store, // term level statistic
position::value_t // expected term position
> term_stats_t;
typedef std::vector<term_stats_t> phrase_stats_t;
typedef std::vector<position::value_t> positions_t;
DECLARE_SHARED_PTR(phrase_query);
phrase_query(states_t&& states, phrase_stats_t&& stats)
: states_(std::move(states)),
stats_(std::move(stats)) {
phrase_query(
states_t&& states,
positions_t&& positions,
attribute_store&& stats
) NOEXCEPT
: prepared(std::move(stats)),
states_(std::move(states)),
positions_(std::move(positions)) {
}
using filter::prepared::execute;
@ -102,6 +103,7 @@ class phrase_query : public filter::prepared {
const attribute_view& /*ctx*/) const override {
// get phrase state for the specified reader
auto phrase_state = states_.find(rdr);
if (!phrase_state) {
// invalid state
return doc_iterator::empty();
@ -110,7 +112,7 @@ class phrase_query : public filter::prepared {
// get features required for query & order
auto features = ord.features() | by_phrase::required();
phrase_iterator::doc_iterators_t itrs;
conjunction::doc_iterators_t itrs;
itrs.reserve(phrase_state->terms.size());
phrase_iterator::positions_t positions;
@ -118,7 +120,8 @@ class phrase_query : public filter::prepared {
// find term using cached state
auto terms = phrase_state->reader->iterator();
auto term_stats = stats_.begin();
auto position = positions_.begin();
for (auto& term_state : phrase_state->terms) {
// use bytes_ref::blank here since we do not need just to "jump"
// to cached state, and we are not interested in term value itself */
@ -126,38 +129,35 @@ class phrase_query : public filter::prepared {
return doc_iterator::empty();
}
// get postings
auto docs = terms->postings(features);
auto docs = terms->postings(features); // postings
auto& pos = docs->attributes().get<irs::position>(); // needed postings attributes
// get needed postings attributes
auto& pos = docs->attributes().get<position>();
if (!pos) {
// positions not found
return doc_iterator::empty();
}
positions.emplace_back(std::ref(*pos), term_stats->second);
positions.emplace_back(std::ref(*pos), *position);
// add base iterator
itrs.emplace_back(doc_iterator::make<basic_doc_iterator>(
rdr,
*phrase_state->reader,
term_stats->first,
std::move(docs),
ord,
term_state.second
));
itrs.emplace_back(std::move(docs));
++term_stats;
++position;
}
return make_conjunction<phrase_iterator>(
std::move(itrs), ord, std::move(positions)
return std::make_shared<phrase_iterator>(
std::move(itrs),
std::move(positions),
rdr,
*phrase_state->reader,
attributes(),
ord
);
}
private:
states_t states_;
phrase_stats_t stats_;
positions_t positions_;
}; // phrase_query
// -----------------------------------------------------------------------------
@ -229,6 +229,7 @@ filter::prepared::ptr by_phrase::prepare(
for (const auto& sr : rdr) {
// get term dictionary for field
const term_reader* tr = sr.field(field);
if (!tr) {
continue;
}
@ -282,27 +283,28 @@ filter::prepared::ptr by_phrase::prepare(
size_t base_offset = first_pos();
// finish stats
phrase_query::phrase_stats_t stats(phrase_.size());
auto stat_itr = stats.begin();
attribute_store attrs; // aggregated phrase stats
phrase_query::positions_t positions(phrase_.size());
auto term_itr = term_stats.begin();
auto pos_itr = positions.begin();
assert(term_stats.size() == phrase_.size()); // initialized above
for(auto& term: phrase_) {
term_itr->finish(stat_itr->first, rdr);
stat_itr->second = position::value_t(term.first - base_offset);
++stat_itr;
term_itr->finish(attrs, rdr);
*pos_itr = position::value_t(term.first - base_offset);
++pos_itr;
++term_itr;
}
auto q = memory::make_shared<phrase_query>(
std::move(phrase_states),
std::move(stats)
);
// apply boost
irs::boost::apply(q->attributes(), this->boost() * boost);
irs::boost::apply(attrs, this->boost() * boost);
return IMPLICIT_MOVE_WORKAROUND(q);
return memory::make_shared<phrase_query>(
std::move(phrase_states),
std::move(positions),
std::move(attrs)
);
}
NS_END // ROOT

59
3rdParty/iresearch/core/search/phrase_iterator.hpp vendored
View File

@ -29,7 +29,9 @@
NS_ROOT
class phrase_iterator final : public conjunction {
// implementation is optimized for frequency based similarity measures
// for generic implementation see a03025accd8b84a5f8ecaaba7412fc92a1636be3
class phrase_iterator final : public doc_iterator_base {
public:
typedef std::pair<
position::ref, // position attribute
@ -39,32 +41,53 @@ class phrase_iterator final : public conjunction {
phrase_iterator(
conjunction::doc_iterators_t&& itrs,
const order::prepared& ord,
positions_t&& pos)
: conjunction(std::move(itrs), ord),
pos_(std::move(pos)) {
positions_t&& pos,
const sub_reader& segment,
const term_reader& field,
const attribute_store& stats,
const order::prepared& ord
) : doc_iterator_base(ord),
approx_(std::move(itrs)),
pos_(std::move(pos)) {
assert(!pos_.empty()); // must not be empty
assert(0 == pos_.front().second); // lead offset is always 0
// add phrase frequency
conjunction::attrs_.emplace(phrase_freq_);
// FIXME find a better estimation
// estimate iterator
estimate([this](){ return irs::cost::extract(approx_.attributes()); });
// set attributes
attrs_.emplace(phrase_freq_); // phrase frequency
attrs_.emplace(doc_); // document (required by scorers)
// set scorers
scorers_ = ord_->prepare_scorers(segment, field, stats, attributes());
prepare_score([this](byte_type* score) { scorers_.score(*ord_, score); });
}
virtual doc_id_t value() const override {
return approx_.value();
}
virtual bool next() override {
bool next = false;
while((next = conjunction::next()) && !(phrase_freq_.value = phrase_freq())) {}
while ((next = approx_.next()) && !(phrase_freq_.value = phrase_freq())) {}
doc_.value = approx_.value();
return next;
}
virtual doc_id_t seek(doc_id_t target) override {
const auto doc = conjunction::seek(target);
doc_.value = approx_.seek(target);
if (type_limits<type_t::doc_id_t>::eof(doc) || (phrase_freq_.value = phrase_freq())) {
return doc;
if (type_limits<type_t::doc_id_t>::eof(doc_.value) || (phrase_freq_.value = phrase_freq())) {
return doc_.value;
}
next();
return this->value();
return value();
}
private:
@ -111,13 +134,13 @@ class phrase_iterator final : public conjunction {
return freq;
}
// a value representing the freqency of the phrase in the document
// FIXME TODO: should be used to modify scoring by supporting scorers
frequency phrase_freq_;
positions_t pos_;
order::prepared::scorers scorers_;
conjunction approx_; // first approximation (conjunction over all words in a phrase)
document doc_; // document itself
frequency phrase_freq_; // freqency of the phrase in a document
positions_t pos_; // list of desired positions along with corresponding attributes
}; // phrase_iterator
NS_END // ROOT
#endif
#endif

2
3rdParty/iresearch/core/search/prefix_filter.cpp vendored
View File

@ -97,7 +97,7 @@ filter::prepared::ptr by_prefix::prepare(
// apply boost
irs::boost::apply(q->attributes(), this->boost() * boost);
return IMPLICIT_MOVE_WORKAROUND(q);
return q;
}
DEFINE_FILTER_TYPE(by_prefix)

2
3rdParty/iresearch/core/search/range_filter.cpp vendored
View File

@ -197,7 +197,7 @@ filter::prepared::ptr by_range::prepare(
// apply boost
irs::boost::apply(q->attributes(), this->boost() * boost);
return IMPLICIT_MOVE_WORKAROUND(q);
return q;
}
NS_END // ROOT

2
3rdParty/iresearch/core/search/same_position_filter.cpp vendored
View File

@ -336,7 +336,7 @@ filter::prepared::ptr by_same_position::prepare(
// apply boost
irs::boost::apply(q->attributes(), this->boost() * boost);
return IMPLICIT_MOVE_WORKAROUND(q);
return q;
}
NS_END // ROOT

32
3rdParty/iresearch/core/search/tfidf.cpp vendored
View File

@ -158,17 +158,17 @@ const frequency EMPTY_FREQ;
struct idf final : basic_stored_attribute<float_t> {
DECLARE_ATTRIBUTE_TYPE();
DECLARE_FACTORY();
idf() : basic_stored_attribute(1.f) { }
idf() : basic_stored_attribute(0.f) { }
void clear() { value = 1.f; }
void clear() { value = 0.f; }
};
DEFINE_ATTRIBUTE_TYPE(iresearch::tfidf::idf);
DEFINE_ATTRIBUTE_TYPE(irs::tfidf::idf);
DEFINE_FACTORY_DEFAULT(idf);
typedef tfidf_sort::score_t score_t;
class collector final : public iresearch::sort::collector {
class collector final : public irs::sort::collector {
public:
explicit collector(bool normalize) NOEXCEPT
: normalize_(normalize) {
@ -179,7 +179,7 @@ class collector final : public iresearch::sort::collector {
const term_reader& field,
const attribute_view& term_attrs
) override {
auto& meta = term_attrs.get<iresearch::term_meta>();
auto& meta = term_attrs.get<irs::term_meta>();
docs_with_field += field.docs_count();
@ -190,10 +190,10 @@ class collector final : public iresearch::sort::collector {
virtual void finish(
attribute_store& filter_attrs,
const iresearch::index_reader& /*index*/
const irs::index_reader& /*index*/
) override {
auto& idf = filter_attrs.emplace<tfidf::idf>();
idf->value = float_t(std::log((docs_with_field + 1) / double_t(docs_with_term + 1)) + 1.0);
idf->value += float_t(std::log((docs_with_field + 1) / double_t(docs_with_term + 1)) + 1.0);
assert(idf->value >= 0);
// add norm attribute if requested
@ -208,12 +208,12 @@ class collector final : public iresearch::sort::collector {
bool normalize_;
}; // collector
class scorer : public iresearch::sort::scorer_base<tfidf::score_t> {
class scorer : public irs::sort::scorer_base<tfidf::score_t> {
public:
DEFINE_FACTORY_INLINE(scorer);
scorer(
iresearch::boost::boost_t boost,
irs::boost::boost_t boost,
const tfidf::idf* idf,
const frequency* freq) NOEXCEPT
: idf_(boost * (idf ? idf->value : 1.f)),
@ -240,8 +240,8 @@ class norm_scorer final : public scorer {
DEFINE_FACTORY_INLINE(norm_scorer);
norm_scorer(
const iresearch::norm* norm,
iresearch::boost::boost_t boost,
const irs::norm* norm,
irs::boost::boost_t boost,
const tfidf::idf* idf,
const frequency* freq) NOEXCEPT
: scorer(boost, idf, freq),
@ -254,10 +254,10 @@ class norm_scorer final : public scorer {
}
private:
const iresearch::norm* norm_;
const irs::norm* norm_;
}; // norm_scorer
class sort final: iresearch::sort::prepared_basic<tfidf::score_t> {
class sort final: irs::sort::prepared_basic<tfidf::score_t> {
public:
DEFINE_FACTORY_INLINE(prepared);
@ -275,7 +275,7 @@ class sort final: iresearch::sort::prepared_basic<tfidf::score_t> {
}
virtual collector::ptr prepare_collector() const override {
return iresearch::sort::collector::make<tfidf::collector>(normalize_);
return irs::sort::collector::make<tfidf::collector>(normalize_);
}
virtual scorer::ptr prepare_scorer(
@ -313,10 +313,10 @@ class sort final: iresearch::sort::prepared_basic<tfidf::score_t> {
NS_END // tfidf
DEFINE_SORT_TYPE_NAMED(iresearch::tfidf_sort, "tfidf");
DEFINE_SORT_TYPE_NAMED(irs::tfidf_sort, "tfidf");
DEFINE_FACTORY_DEFAULT(irs::tfidf_sort);
tfidf_sort::tfidf_sort(bool normalize)
tfidf_sort::tfidf_sort(bool normalize) NOEXCEPT
: sort(tfidf_sort::type()),
normalize_(normalize) {
}

6
3rdParty/iresearch/core/search/tfidf.hpp vendored
View File

@ -41,11 +41,11 @@ public:
typedef float_t score_t;
explicit tfidf_sort(bool normalize = WITH_NORMS());
explicit tfidf_sort(bool normalize = WITH_NORMS()) NOEXCEPT;
static void init(); // for trigering registration in a static build
bool normalize() const { return normalize_; }
void normalize(bool value) { normalize_ = value; }
bool normalize() const NOEXCEPT { return normalize_; }
void normalize(bool value) NOEXCEPT { normalize_ = value; }
virtual sort::prepared::ptr prepare() const override;

19
3rdParty/iresearch/core/shared.hpp vendored
View File

@ -137,23 +137,6 @@
#define MAX_ALIGN_T std::max_align_t
#endif
// GCC before v5 does not implicitly call the move constructor on local values
// returned from functions, e.g. std::unique_ptr
//
// MSVC2013 doesn't support c++11 in a proper way
// sometimes it can't choose move constructor for
// move-only types while returning a value.
// The following macro tries to avoid potential
// performance problems on other compilers since
// 'return std::move(x)' prevents such compiler
// optimizations like 'copy elision'
#if (defined(__GNUC__) && (__GNUC__ < 5)) \
|| (defined(_MSC_VER) && _MSC_VER < 1900)
#define IMPLICIT_MOVE_WORKAROUND(x) std::move(x)
#else
#define IMPLICIT_MOVE_WORKAROUND(x) x
#endif
// hook for GCC 8.1/8.2 optimized code
// these versions produce incorrect code when inlining optimizations are enabled
#if defined(__OPTIMIZE__) && defined(__GNUC__) \
@ -175,7 +158,7 @@
|| ((_MSC_FULL_VER >= 191326128) && (_MSC_FULL_VER <= 191326132)) \
|| ((_MSC_FULL_VER >= 191426430) && (_MSC_FULL_VER <= 191426433)) \
|| ((_MSC_FULL_VER >= 191526726) && (_MSC_FULL_VER <= 191526732)) \
|| ((_MSC_FULL_VER >= 191627023) && (_MSC_FULL_VER <= 191627023)))
|| ((_MSC_FULL_VER >= 191627023) && (_MSC_FULL_VER <= 191627025)))
#define MSVC2017_345678_OPTIMIZED_WORKAROUND(...) __VA_ARGS__
#else
#define MSVC2017_345678_OPTIMIZED_WORKAROUND(...)

2
3rdParty/iresearch/tests/CMakeLists.txt vendored
View File

@ -212,7 +212,7 @@ target_link_libraries(${IResearchTests_TARGET_NAME}-shared
target_link_libraries(${IResearchTests_TARGET_NAME}-static
${GCOV_LIBRARY}
${IResearch_TARGET_NAME}-static
${IResearch_TARGET_NAME}-static-allinone
${GTEST_STATIC_LIBS}
${PTHREAD_LIBRARY}
)

1165
3rdParty/iresearch/tests/index/consolidation_policy_tests.cpp vendored
View File

File diff suppressed because it is too large Load Diff

52
3rdParty/iresearch/tests/index/index_death_tests.cpp vendored
View File

@ -2374,7 +2374,7 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
dir.register_failure(failing_directory::Failure::CREATE, "_8.0.sm"); // segment meta
// write index
irs::index_writer::options opts;
irs::index_writer::init_options opts;
opts.segment_docs_max = 1; // flush every 2nd document
auto writer = irs::index_writer::make(dir, codec, irs::OM_CREATE, opts);
@ -2422,7 +2422,7 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
dir.register_failure(failing_directory::Failure::CREATE, "_8.0.sm"); // segment meta
// write index
irs::index_writer::options opts;
irs::index_writer::init_options opts;
opts.segment_docs_max = 1; // flush every 2nd document
auto writer = irs::index_writer::make(dir, codec, irs::OM_CREATE, opts);
@ -2485,6 +2485,26 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
ASSERT_EQ("C", irs::to_string<irs::string_ref>(actual_value.c_str())); // 'name' value in doc3
ASSERT_FALSE(docsItr->next());
}
}
TEST(index_death_test_formats_10, columnstore_creation_fail_implicit_segment_flush) {
const auto all_features = irs::flags{
irs::document::type(),
irs::frequency::type(),
irs::position::type(),
irs::payload::type(),
irs::offset::type()
};
tests::json_doc_generator gen(
test_base::resource("simple_sequential.json"),
&tests::payloaded_json_field_factory
);
const auto* doc1 = gen.next();
const auto* doc2 = gen.next();
auto codec = irs::formats::get("1_0");
ASSERT_NE(nullptr, codec);
// columnstore creation failure
{
@ -2492,7 +2512,7 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
failing_directory dir(impl);
// write index
irs::index_writer::options opts;
irs::index_writer::init_options opts;
opts.segment_docs_max = 1; // flush every 2nd document
auto writer = irs::index_writer::make(dir, codec, irs::OM_CREATE, opts);
@ -2548,6 +2568,26 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
ASSERT_EQ("A", irs::to_string<irs::string_ref>(actual_value.c_str())); // 'name' value in doc3
ASSERT_FALSE(docsItr->next());
}
}
TEST(index_death_test_formats_10, columnstore_creation_sync_fail_implicit_segment_flush) {
const auto all_features = irs::flags{
irs::document::type(),
irs::frequency::type(),
irs::position::type(),
irs::payload::type(),
irs::offset::type()
};
tests::json_doc_generator gen(
test_base::resource("simple_sequential.json"),
&tests::payloaded_json_field_factory
);
const auto* doc1 = gen.next();
const auto* doc2 = gen.next();
auto codec = irs::formats::get("1_0");
ASSERT_NE(nullptr, codec);
// columnstore creation + sync failures
{
@ -2555,7 +2595,7 @@ TEST(index_death_test_formats_10, segment_components_creation_fail_implicit_segm
failing_directory dir(impl);
// write index
irs::index_writer::options opts;
irs::index_writer::init_options opts;
opts.segment_docs_max = 1; // flush every 2nd document
auto writer = irs::index_writer::make(dir, codec, irs::OM_CREATE, opts);
@ -2943,3 +2983,7 @@ TEST(index_death_test_formats_10, postings_reopen_fail) {
ASSERT_FALSE(live_docs->next());
ASSERT_EQ(irs::type_limits<irs::type_t::doc_id_t>::eof(), live_docs->value());
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

6
3rdParty/iresearch/tests/index/index_profile_tests.cpp vendored
View File

@ -87,7 +87,7 @@ class index_test_case_base: public index_test_base {
std::mutex commit_mutex;
if (!writer) {
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_count_max = 8; // match original implementation or may run out of file handles (e.g. MacOS/Travis)
writer = open_writer(irs::OM_CREATE, options);
}
@ -386,7 +386,7 @@ class index_test_case_base: public index_test_base {
size_t commit_interval
) {
auto* directory = &dir();
irs::index_writer::options options;
irs::index_writer::init_options options;
std::atomic<bool> working(true);
std::atomic<size_t> writer_commit_count(0);
@ -420,7 +420,7 @@ class index_test_case_base: public index_test_base {
void profile_bulk_index_dedicated_consolidate(size_t num_threads, size_t batch_size, size_t consolidate_interval) {
const auto policy = irs::index_utils::consolidation_policy(irs::index_utils::consolidate_count());
auto* directory = &dir();
irs::index_writer::options options;
irs::index_writer::init_options options;
std::atomic<bool> working(true);
irs::async_utils::thread_pool thread_pool(2, 2);

255
3rdParty/iresearch/tests/index/index_tests.cpp vendored
View File

@ -672,7 +672,7 @@ class index_test_case_base : public tests::index_test_base {
ASSERT_EQ(&irs::memory_allocator::global(), &irs::directory_utils::get_allocator(dir));
// open writer
irs::index_writer::options options;
irs::index_writer::init_options options;
options.memory_pool_size = 42;
auto writer = irs::index_writer::make(dir, codec(), irs::OM_CREATE, options);
ASSERT_NE(nullptr, writer);
@ -732,13 +732,13 @@ class index_test_case_base : public tests::index_test_base {
{
// open writer with NOLOCK hint
irs::index_writer::options options0;
irs::index_writer::init_options options0;
options0.lock_repository = false;
auto writer0 = irs::index_writer::make(dir(), codec(), irs::OM_CREATE, options0);
ASSERT_NE(nullptr, writer0);
// can open another writer at the same time on the same directory
irs::index_writer::options options1;
irs::index_writer::init_options options1;
options1.lock_repository = false;
auto writer1 = irs::index_writer::make(dir(), codec(), irs::OM_CREATE, options1);
ASSERT_NE(nullptr, writer1);
@ -749,7 +749,7 @@ class index_test_case_base : public tests::index_test_base {
{
// open writer with NOLOCK hint
irs::index_writer::options options0;
irs::index_writer::init_options options0;
options0.lock_repository = false;
auto writer0 = irs::index_writer::make(dir(), codec(), irs::OM_CREATE, options0);
ASSERT_NE(nullptr, writer0);
@ -764,7 +764,7 @@ class index_test_case_base : public tests::index_test_base {
{
// open writer with NOLOCK hint
irs::index_writer::options options0;
irs::index_writer::init_options options0;
options0.lock_repository = false;
auto writer0 = irs::index_writer::make(dir(), codec(), irs::OM_CREATE, options0);
ASSERT_NE(nullptr, writer0);
@ -12215,7 +12215,7 @@ TEST_F(memory_index_test, document_context) {
// rollback inserts split over multiple segment_writers
{
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -12367,7 +12367,7 @@ TEST_F(memory_index_test, document_context) {
{
auto query_doc1 = irs::iql::query_builder().build("name==A", std::locale::classic());
auto query_doc2 = irs::iql::query_builder().build("name==B", std::locale::classic());
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -12529,7 +12529,7 @@ TEST_F(memory_index_test, document_context) {
{
auto query_doc1 = irs::iql::query_builder().build("name==A", std::locale::classic());
auto query_doc2 = irs::iql::query_builder().build("name==B", std::locale::classic());
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -12695,7 +12695,7 @@ TEST_F(memory_index_test, document_context) {
{
auto query_doc1 = irs::iql::query_builder().build("name==A", std::locale::classic());
auto query_doc2 = irs::iql::query_builder().build("name==B", std::locale::classic());
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -12771,7 +12771,7 @@ TEST_F(memory_index_test, document_context) {
// segment flush due to memory bytes limit (same flush_context)
{
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_memory_max = 1; // arbitaty size < 1 document (first doc will always aquire a new segment_writer)
auto writer = open_writer(irs::OM_CREATE, options);
@ -12830,7 +12830,7 @@ TEST_F(memory_index_test, document_context) {
// segment flush due to memory bytes limit (split over different flush_contexts)
{
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_memory_max = 1; // arbitaty size < 1 document (first doc will always aquire a new segment_writer)
auto writer = open_writer(irs::OM_CREATE, options);
@ -12899,7 +12899,7 @@ TEST_F(memory_index_test, document_context) {
// segment flush due to document count limit (same flush_context)
{
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -12958,7 +12958,7 @@ TEST_F(memory_index_test, document_context) {
// segment flush due to document count limit (split over different flush_contexts)
{
irs::index_writer::options options;
irs::index_writer::init_options options;
options.segment_docs_max = 1; // each doc will have its own segment
auto writer = open_writer(irs::OM_CREATE, options);
@ -20064,6 +20064,233 @@ TEST_F(memory_index_test, segment_consolidate_policy) {
}
}
TEST_F(memory_index_test, segment_options) {
tests::json_doc_generator gen(
resource("simple_sequential.json"),
[] (tests::document& doc, const std::string& name, const tests::json_doc_generator::json_value& data) {
if (data.is_string()) {
doc.insert(std::make_shared<tests::templates::string_field>(
irs::string_ref(name),
data.str
));
}
});
tests::document const* doc1 = gen.next();
tests::document const* doc2 = gen.next();
// segment_count_max
{
auto writer = open_writer();
auto ctx = writer->documents(); // hold a single segment
{
auto doc = ctx.insert();
ASSERT_TRUE(
doc.insert(irs::action::index, doc1->indexed.begin(), doc1->indexed.end())
&& doc.insert(irs::action::store, doc1->stored.begin(), doc1->stored.end())
);
}
irs::index_writer::segment_options options;
options.segment_count_max = 1;
writer->options(options);
std::condition_variable cond;
std::mutex mutex;
SCOPED_LOCK_NAMED(mutex, lock);
std::atomic<bool> stop(false);
std::thread thread([&writer, &doc2, &cond, &mutex, &stop]()->void {
ASSERT_TRUE(insert(*writer,
doc2->indexed.begin(), doc2->indexed.end(),
doc2->stored.begin(), doc2->stored.end()
));
stop = true;
SCOPED_LOCK(mutex);
cond.notify_all();
});
auto result = cond.wait_for(lock, std::chrono::milliseconds(1000)); // assume thread blocks in 1000ms
// MSVC 2015/2017 seems to sporadically notify condition variables without explicit request
MSVC2015_ONLY(while(!stop && result == std::cv_status::no_timeout) result = cond.wait_for(lock, std::chrono::milliseconds(1000)));
MSVC2017_ONLY(while(!stop && result == std::cv_status::no_timeout) result = cond.wait_for(lock, std::chrono::milliseconds(1000)));
ASSERT_EQ(std::cv_status::timeout, result);
// ^^^ expecting timeout because pool should block indefinitely
{ irs::index_writer::documents_context(std::move(ctx)); } // force flush of documents(), i.e. ulock segment
//ASSERT_EQ(std::cv_status::no_timeout, cond.wait_for(lock, std::chrono::milliseconds(1000)));
lock.unlock();
thread.join();
ASSERT_TRUE(stop);
writer->commit();
auto reader = iresearch::directory_reader::open(dir(), codec());
ASSERT_EQ(1, reader.size());
// check only segment
{
std::unordered_set<irs::string_ref> expectedName = { "A", "B" };
auto& segment = reader[0];
const auto* column = segment.column_reader("name");
ASSERT_NE(nullptr, column);
auto values = column->values();
ASSERT_EQ(expectedName.size(), segment.docs_count()); // total count of documents
auto terms = segment.field("same");
ASSERT_NE(nullptr, terms);
auto termItr = terms->iterator();
ASSERT_TRUE(termItr->next());
irs::bytes_ref actual_value;
for (auto docsItr = termItr->postings(iresearch::flags()); docsItr->next();) {
ASSERT_TRUE(values(docsItr->value(), actual_value));
ASSERT_EQ(1, expectedName.erase(irs::to_string<irs::string_ref>(actual_value.c_str())));
}
ASSERT_TRUE(expectedName.empty());
}
}
// segment_docs_max
{
auto writer = open_writer();
ASSERT_TRUE(insert(*writer,
doc1->indexed.begin(), doc1->indexed.end(),
doc1->stored.begin(), doc1->stored.end()
));
irs::index_writer::segment_options options;
options.segment_docs_max = 1;
writer->options(options);
ASSERT_TRUE(insert(*writer,
doc2->indexed.begin(), doc2->indexed.end(),
doc2->stored.begin(), doc2->stored.end()
));
writer->commit();
auto reader = iresearch::directory_reader::open(dir(), codec());
ASSERT_EQ(2, reader.size()); // 1+2
// check 1st segment
{
std::unordered_set<irs::string_ref> expectedName = { "A" };
auto& segment = reader[0];
const auto* column = segment.column_reader("name");
ASSERT_NE(nullptr, column);
auto values = column->values();
ASSERT_EQ(expectedName.size(), segment.docs_count()); // total count of documents
auto terms = segment.field("same");
ASSERT_NE(nullptr, terms);
auto termItr = terms->iterator();
ASSERT_TRUE(termItr->next());
irs::bytes_ref actual_value;
for (auto docsItr = termItr->postings(iresearch::flags()); docsItr->next();) {
ASSERT_TRUE(values(docsItr->value(), actual_value));
ASSERT_EQ(1, expectedName.erase(irs::to_string<irs::string_ref>(actual_value.c_str())));
}
ASSERT_TRUE(expectedName.empty());
}
// check 2nd segment
{
std::unordered_set<irs::string_ref> expectedName = { "B" };
auto& segment = reader[1];
const auto* column = segment.column_reader("name");
ASSERT_NE(nullptr, column);
auto values = column->values();
ASSERT_EQ(expectedName.size(), segment.docs_count()); // total count of documents
auto terms = segment.field("same");
ASSERT_NE(nullptr, terms);
auto termItr = terms->iterator();
ASSERT_TRUE(termItr->next());
irs::bytes_ref actual_value;
for (auto docsItr = termItr->postings(iresearch::flags()); docsItr->next();) {
ASSERT_TRUE(values(docsItr->value(), actual_value));
ASSERT_EQ(1, expectedName.erase(irs::to_string<irs::string_ref>(actual_value.c_str())));
}
ASSERT_TRUE(expectedName.empty());
}
}
// segment_memory_max
{
auto writer = open_writer();
ASSERT_TRUE(insert(*writer,
doc1->indexed.begin(), doc1->indexed.end(),
doc1->stored.begin(), doc1->stored.end()
));
irs::index_writer::segment_options options;
options.segment_memory_max = 1;
writer->options(options);
ASSERT_TRUE(insert(*writer,
doc2->indexed.begin(), doc2->indexed.end(),
doc2->stored.begin(), doc2->stored.end()
));
writer->commit();
auto reader = iresearch::directory_reader::open(dir(), codec());
ASSERT_EQ(2, reader.size()); // 1+2
// check 1st segment
{
std::unordered_set<irs::string_ref> expectedName = { "A" };
auto& segment = reader[0];
const auto* column = segment.column_reader("name");
ASSERT_NE(nullptr, column);
auto values = column->values();
ASSERT_EQ(expectedName.size(), segment.docs_count()); // total count of documents
auto terms = segment.field("same");
ASSERT_NE(nullptr, terms);
auto termItr = terms->iterator();
ASSERT_TRUE(termItr->next());
irs::bytes_ref actual_value;
for (auto docsItr = termItr->postings(iresearch::flags()); docsItr->next();) {
ASSERT_TRUE(values(docsItr->value(), actual_value));
ASSERT_EQ(1, expectedName.erase(irs::to_string<irs::string_ref>(actual_value.c_str())));
}
ASSERT_TRUE(expectedName.empty());
}
// check 2nd segment
{
std::unordered_set<irs::string_ref> expectedName = { "B" };
auto& segment = reader[1];
const auto* column = segment.column_reader("name");
ASSERT_NE(nullptr, column);
auto values = column->values();
ASSERT_EQ(expectedName.size(), segment.docs_count()); // total count of documents
auto terms = segment.field("same");
ASSERT_NE(nullptr, terms);
auto termItr = terms->iterator();
ASSERT_TRUE(termItr->next());
irs::bytes_ref actual_value;
for (auto docsItr = termItr->postings(iresearch::flags()); docsItr->next();) {
ASSERT_TRUE(values(docsItr->value(), actual_value));
ASSERT_EQ(1, expectedName.erase(irs::to_string<irs::string_ref>(actual_value.c_str())));
}
ASSERT_TRUE(expectedName.empty());
}
}
}
// ----------------------------------------------------------------------------
// --SECTION-- fs_directory + iresearch_format_10
// ----------------------------------------------------------------------------
@ -20324,4 +20551,4 @@ TEST_F(mmap_index_test, writer_close) {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

4
3rdParty/iresearch/tests/index/index_tests.hpp vendored
View File

@ -163,14 +163,14 @@ class index_test_base : public virtual test_base {
irs::index_writer::ptr open_writer(
irs::directory& dir,
irs::OpenMode mode = irs::OM_CREATE,
const irs::index_writer::options& options = {}
const irs::index_writer::init_options& options = {}
) {
return irs::index_writer::make(dir, codec_, mode, options);
}
irs::index_writer::ptr open_writer(
irs::OpenMode mode = irs::OM_CREATE,
const irs::index_writer::options& options = {}
const irs::index_writer::init_options& options = {}
) {
return irs::index_writer::make(*dir_, codec_, mode, options);
}

1
3rdParty/iresearch/tests/index/transaction_store_tests.cpp vendored
View File

@ -5682,7 +5682,6 @@ TEST_F(transaction_store_tests, read_reopen) {
ASSERT_EQ(1, reader0.size());
ASSERT_NE(reader0.begin(), reader0.end());
ASSERT_EQ(&*(reader.begin()), &*(reader0.begin()));
ASSERT_EQ(&*(reader.end()), &*(reader0.end()));
}
// write 2nd generation

2
3rdParty/iresearch/tests/search/bm25_test.cpp vendored
View File

@ -239,8 +239,8 @@ TEST_F(bm25_test, test_phrase) {
std::multimap<irs::bstring, std::string, decltype(comparer)> sorted(comparer);
std::vector<std::string> expected{
"P", // jumps high jumps left jumps right jumps down jumps back
"O", // jumps high jumps high hotdog
"P", // jumps high jumps left jumps right jumps down jumps back
"Q", // jumps high jumps left jumps right jumps down walks back
"R" // jumps high jumps left jumps right walks down walks back
};

6
3rdParty/iresearch/tests/search/tfidf_test.cpp vendored
View File

@ -248,10 +248,10 @@ TEST_F(tfidf_test, test_phrase) {
std::multimap<irs::bstring, std::string, decltype(comparer)> sorted(comparer);
std::vector<std::string> expected{
std::vector<std::string> expected {
"O", // jumps high jumps high hotdog
"P", // jumps high jumps left jumps right jumps down jumps back
"Q", // jumps high jumps left jumps right jumps down walks back
"O", // jumps high jumps high hotdog
"R" // jumps high jumps left jumps right walks down walks back
};
@ -733,4 +733,4 @@ TEST_F(tfidf_test, test_order) {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

4
3rdParty/iresearch/utils/CMakeLists.txt vendored
View File

@ -43,7 +43,7 @@ set_target_properties(${IResearchUtil_TARGET_NAME}
)
target_link_libraries(${IResearchUtil_TARGET_NAME}
${IResearch_TARGET_NAME}-static
${IResearch_TARGET_NAME}-static-allinone
${PTHREAD_LIBRARY}
${ATOMIC_LIBRARY}
)
@ -102,7 +102,7 @@ if (NOT MSVC AND GCC_VERSION VERSION_LESS 4.9)
endif()
target_link_libraries(${IResearchBencmarks_TARGET_NAME}
${IResearch_TARGET_NAME}-static
${IResearch_TARGET_NAME}-static-allinone
${PTHREAD_LIBRARY}
${ATOMIC_LIBRARY}
${REGEX_LIBRARY}

4
arangod/IResearch/IResearchLink.cpp
View File

@ -846,7 +846,7 @@ arangodb::Result IResearchLink::initDataStore() {
);
}
irs::index_writer::options options;
irs::index_writer::init_options options;
options.lock_repository = false; // do not lock index, ArangoDB has it's own lock
@ -1096,7 +1096,7 @@ size_t IResearchLink::memory() const {
}
bool IResearchLink::properties(
irs::index_writer::segment_limits const& properties
irs::index_writer::segment_options const& properties
) {
// FIXME TODO update the data-store options
return true;

2
arangod/IResearch/IResearchLink.h
View File

@ -176,7 +176,7 @@ class IResearchLink {
/// @brief update runtine data processing properties (not persisted)
/// @return success
//////////////////////////////////////////////////////////////////////////////
bool properties(irs::index_writer::segment_limits const& properties);
bool properties(irs::index_writer::segment_options const& properties);
////////////////////////////////////////////////////////////////////////////////
/// @brief remove an ArangoDB document from an iResearch View

8
arangod/IResearch/IResearchView.cpp
View File

@ -758,11 +758,9 @@ bool IResearchView::link(AsyncLinkPtr const& link) {
SCOPED_LOCK(mutex);
auto itr = _links.find(cid);
irs::index_writer::options options;
options.segment_count_max = _meta._writebufferActive;
options.segment_memory_max = _meta._writebufferSizeMax;
options.segment_pool_size = _meta._writebufferIdle;
irs::index_writer::segment_limits properties(options);
irs::index_writer::segment_options properties;
properties.segment_count_max = _meta._writebufferActive;
properties.segment_memory_max = _meta._writebufferSizeMax;
if (itr == _links.end()) {
_links.emplace(cid, link);