issue 373.4: address issue with MSVC picking incorrect function, minor formatting changes, backport from iresearch upstream: ensure id_to_column contains enough slots for all column ids, address another issue with _T(...) macro not working when _MBCS is defined on MSVC (#5310)

3rdParty/iresearch/core/formats/formats.hpp

@@ -284,7 +284,8 @@ struct IRESEARCH_API column_meta_reader {
   virtual bool prepare(
     const directory& dir, 
     const segment_meta& meta,
-    /*out*/ field_id& count
+    size_t& count, // out parameter
+    field_id& max_id // out parameter
   ) = 0;
   // returns false if there is no more data to read
   virtual bool read(column_meta& column) = 0;

3rdParty/iresearch/core/formats/formats_10.cpp

@@ -1591,7 +1591,8 @@ class meta_writer final : public iresearch::column_meta_writer {
 
  private:
   index_output::ptr out_;
-  field_id count_{}; // number of written objects
+  size_t count_{}; // number of written objects
+  field_id max_id_{}; // the highest column id written (optimization for vector resize on read to highest id)
 }; // meta_writer 
 
 MSVC2015_ONLY(__pragma(warning(push)))
@@ -1626,10 +1627,12 @@ void meta_writer::write(const std::string& name, field_id id) {
   out_->write_vlong(id);
   write_string(*out_, name);
   ++count_;
+  max_id_ = std::max(max_id_, id);
 }
 
 void meta_writer::flush() {
   out_->write_long(count_); // write total number of written objects
+  out_->write_long(max_id_); // write highest column id written
   format_utils::write_footer(*out_);
   out_.reset();
   count_ = 0;
@@ -1640,19 +1643,22 @@ class meta_reader final : public iresearch::column_meta_reader {
   virtual bool prepare(
     const directory& dir, 
     const segment_meta& meta,
-    field_id& count
+    size_t& count,
+    field_id& max_id
   ) override;
   virtual bool read(column_meta& column) override;
 
  private:
   index_input::ptr in_;
-  field_id count_{0};
+  size_t count_{0};
+  field_id max_id_{0};
 }; // meta_writer
 
 bool meta_reader::prepare(
     const directory& dir,
     const segment_meta& meta,
-    field_id& count
+    size_t& count,
+    field_id& max_id
 ) {
   auto filename = file_name<column_meta_writer>(meta);
 
@@ -1667,11 +1673,11 @@ bool meta_reader::prepare(
 
   const auto checksum = format_utils::checksum(*in);
 
-  in->seek(in->length() - sizeof(field_id) - format_utils::FOOTER_LEN);
-
-  // read number of objects to read
-  count = in->read_long();
-
+  in->seek( // seek to start of meta footer (before count and max_id)
+    in->length() - sizeof(size_t) - sizeof(field_id) - format_utils::FOOTER_LEN
+  );
+  count = in->read_long(); // read number of objects to read
+  max_id = in->read_long(); // read highest column id written
   format_utils::check_footer(*in, checksum);
 
   in->seek(0);
@@ -1685,6 +1691,7 @@ bool meta_reader::prepare(
 
   in_ = std::move(in);
   count_ = count;
+  max_id_ = max_id;
   return true;
 }
 
@@ -1694,9 +1701,12 @@ bool meta_reader::read(column_meta& column) {
   }
 
   const auto id = in_->read_vlong();
+
+  assert(id <= max_id_);
   column.name = read_string<std::string>(*in_);
   column.id = id;
   --count_;
+
   return true;
 }
 

3rdParty/iresearch/core/index/segment_reader.cpp

@@ -149,32 +149,36 @@ bool read_columns_meta(
     std::vector<iresearch::column_meta*>& id_to_column,
     std::unordered_map<iresearch::hashed_string_ref, iresearch::column_meta*>& name_to_column
 ) {
+  size_t count = 0;
+  irs::field_id max_id;
   auto reader = codec.get_column_meta_reader();
-  iresearch::field_id count = 0;
 
-  if (!reader->prepare(dir, meta, count)) {
+  if (!reader->prepare(dir, meta, count, max_id)
+      || max_id >= irs::integer_traits<size_t>::const_max) {
     return false;
   }
 
   columns.reserve(count);
-  id_to_column.resize(count);
+  id_to_column.resize(max_id + 1); // +1 for count
   name_to_column.reserve(count);
 
   for (irs::column_meta col_meta; reader->read(col_meta);) {
     columns.emplace_back(std::move(col_meta));
 
     auto& column = columns.back();
-    id_to_column[column.id] = &column;
-
     const auto res = name_to_column.emplace(
       irs::make_hashed_ref(iresearch::string_ref(column.name), std::hash<irs::string_ref>()),
       &column
     );
 
-    if (!res.second) {
+    assert(column.id < id_to_column.size());
+
+    if (!res.second || id_to_column[column.id]) {
       // duplicate field
       return false;
     }
+
+    id_to_column[column.id] = &column;
   }
 
   if (!std::is_sorted(
@@ -435,4 +439,4 @@ const columnstore_reader::column_reader* segment_reader_impl::column_reader(
     : nullptr;
 }
 
-NS_END
+NS_END

3rdParty/iresearch/core/utils/file_utils.cpp

@@ -600,7 +600,7 @@ bool mtime(time_t& result, int fd) NOEXCEPT {
 handle_t open(const file_path_t path, const file_path_t mode) NOEXCEPT {
   #ifdef _WIN32
     #pragma warning(disable: 4996) // '_wfopen': This function or variable may be unsafe.
-    handle_t handle(::_wfopen(path ? path : _T("NUL:"), mode));
+    handle_t handle(::_wfopen(path ? path : IR_WSTR("NUL:"), mode));
     #pragma warning(default: 4996)
   #else
     handle_t handle(::fopen(path ? path : "/dev/null", mode));

arangod/IResearch/IResearchViewDBServer.cpp

@@ -235,7 +235,8 @@ std::shared_ptr<arangodb::LogicalView> IResearchViewDBServer::ensure(
   static const std::function<bool(irs::string_ref const& key)> acceptor = [](
       irs::string_ref const& key
   )->bool {
-    return key != StaticStrings::CollectionsField && key != StaticStrings::LinksField; // ignored fields
+    return key != StaticStrings::CollectionsField
+      && key != StaticStrings::LinksField; // ignored fields
   };
   arangodb::velocypack::Builder builder;
 

tests/IResearch/AgencyCommManagerMock.cpp

@@ -85,7 +85,7 @@ int EndpointMock::port() const {
 
 GeneralClientConnectionMock::GeneralClientConnectionMock()
   : GeneralClientConnection(&endpoint, 0, 0, 0),
-    nil(file_open((file_path_t)nullptr, "rw")) {
+    nil(file_open((const file_path_t)nullptr, "rw")) {
   _socket.fileDescriptor = file_no(nil.get()); // must be a readable/writable descriptor
 }
 
@@ -185,4 +185,4 @@ void GeneralClientConnectionMapMock::response(
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       END-OF-FILE
-// -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------