Feature/windows utf16 fileaccess (#6534)

3rdParty/rocksdb/CMakeLists.txt

@@ -38,6 +38,7 @@ if (MSVC)
   set(SNAPPY_INCLUDE "${SNAPPY_INCLUDE_DIR}" CACHE PATH "where the wintendo should look for the snappy libs")
   set(SNAPPY_LIB_DEBUG "${CMAKE_BINARY_DIR}/bin/Debug/${SNAPPY_LIB}.lib")
   set(SNAPPY_LIB_RELEASE "${CMAKE_BINARY_DIR}/bin/RelWithDebInfo/${SNAPPY_LIB}.lib")
+  set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "we want to provide utf8 filenames")
 endif ()
 set(USE_RTTI ON CACHE BOOL "enable RTTI")
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${ARANGO_ROCKSDB_VERSION} EXCLUDE_FROM_ALL)

3rdParty/rocksdb/v5.16.X/CMakeLists.txt

@@ -49,6 +49,10 @@ option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
 option(WITH_ZSTD "build with zstd" OFF)
+option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
+if (WITH_WINDOWS_UTF8_FILENAMES)
+  add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
+endif()
 if(MSVC)
   # Defaults currently different for GFLAGS.
   #  We will address find_package work a little later

3rdParty/rocksdb/v5.16.X/port/win/env_win.cc

@@ -102,7 +102,8 @@ WinEnvIO::~WinEnvIO() {
 Status WinEnvIO::DeleteFile(const std::string& fname) {
   Status result;
 
-  BOOL ret = DeleteFileA(fname.c_str());
+  BOOL ret = RX_DeleteFile(RX_FN(fname).c_str());
+
   if(!ret) {
     auto lastError = GetLastError();
     result = IOErrorFromWindowsError("Failed to delete: " + fname,
@@ -114,7 +115,7 @@ Status WinEnvIO::DeleteFile(const std::string& fname) {
 
 Status WinEnvIO::Truncate(const std::string& fname, size_t size) {
   Status s;
-  int result = truncate(fname.c_str(), size);
+  int result = rocksdb::port::Truncate(fname, size);
   if (result != 0) {
     s = IOError("Failed to truncate: " + fname, errno);
   }
@@ -151,8 +152,8 @@ Status WinEnvIO::NewSequentialFile(const std::string& fname,
 
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    hFile = CreateFileA(
+    hFile = RX_CreateFile(
-      fname.c_str(), GENERIC_READ,
+      RX_FN(fname).c_str(), GENERIC_READ,
       FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
       OPEN_EXISTING,  // Original fopen mode is "rb"
       fileFlags, NULL);
@@ -190,7 +191,7 @@ Status WinEnvIO::NewRandomAccessFile(const std::string& fname,
   {
     IOSTATS_TIMER_GUARD(open_nanos);
     hFile =
-      CreateFileA(fname.c_str(), GENERIC_READ,
+      RX_CreateFile(RX_FN(fname).c_str(), GENERIC_READ,
       FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
       NULL, OPEN_EXISTING, fileFlags, NULL);
   }
@@ -217,7 +218,7 @@ Status WinEnvIO::NewRandomAccessFile(const std::string& fname,
           "NewRandomAccessFile failed to map empty file: " + fname, EINVAL);
       }
 
-      HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READONLY,
+      HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READONLY,
         0,  // Whole file at its present length
         0,
         NULL);  // Mapping name
@@ -302,8 +303,8 @@ Status WinEnvIO::OpenWritableFile(const std::string& fname,
   HANDLE hFile = 0;
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    hFile = CreateFileA(
+    hFile = RX_CreateFile(
-      fname.c_str(),
+      RX_FN(fname).c_str(),
       desired_access,  // Access desired
       shared_mode,
       NULL,           // Security attributes
@@ -366,7 +367,7 @@ Status WinEnvIO::NewRandomRWFile(const std::string & fname,
   {
     IOSTATS_TIMER_GUARD(open_nanos);
     hFile =
-      CreateFileA(fname.c_str(),
+      RX_CreateFile(RX_FN(fname).c_str(),
         desired_access,
         shared_mode,
         NULL, // Security attributes
@@ -399,8 +400,8 @@ Status WinEnvIO::NewMemoryMappedFileBuffer(const std::string & fname,
   HANDLE hFile = INVALID_HANDLE_VALUE;
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    hFile = CreateFileA(
+    hFile = RX_CreateFile(
-      fname.c_str(), GENERIC_READ | GENERIC_WRITE,
+      RX_FN(fname).c_str(), GENERIC_READ | GENERIC_WRITE,
       FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
       NULL,
       OPEN_EXISTING,  // Open only if it exists
@@ -432,7 +433,7 @@ Status WinEnvIO::NewMemoryMappedFileBuffer(const std::string & fname,
       "The specified file size does not fit into 32-bit memory addressing: " + fname);
   }
 
-  HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READWRITE,
+  HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READWRITE,
       0,  // Whole file at its present length
       0,
       NULL);  // Mapping name
@@ -483,7 +484,7 @@ Status WinEnvIO::NewDirectory(const std::string& name,
   // 0 - for access means read metadata
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    handle = ::CreateFileA(name.c_str(), 0,
+    handle = RX_CreateFile(RX_FN(name).c_str(), 0,
       FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
       NULL,
       OPEN_EXISTING,
@@ -509,8 +510,7 @@ Status WinEnvIO::FileExists(const std::string& fname) {
   // which is consistent with _access() impl on windows
   // but can be added
   WIN32_FILE_ATTRIBUTE_DATA attrs;
-  if (FALSE == GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard,
+  if (FALSE == RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) {
-    &attrs)) {
     auto lastError = GetLastError();
     switch (lastError) {
     case ERROR_ACCESS_DENIED:
@@ -535,11 +535,12 @@ Status WinEnvIO::GetChildren(const std::string& dir,
   result->clear();
   std::vector<std::string> output;
 
-  WIN32_FIND_DATA data;
+  RX_WIN32_FIND_DATA data;
+  memset(&data, 0, sizeof(data));
   std::string pattern(dir);
   pattern.append("\\").append("*");
 
-  HANDLE handle = ::FindFirstFileExA(pattern.c_str(),
+  HANDLE handle = RX_FindFirstFileEx(RX_FN(pattern).c_str(),
     FindExInfoBasic, // Do not want alternative name
     &data,
     FindExSearchNameMatch,
@@ -572,8 +573,9 @@ Status WinEnvIO::GetChildren(const std::string& dir,
   data.cFileName[MAX_PATH - 1] = 0;
 
   while (true) {
-    output.emplace_back(data.cFileName);
+    auto x = RX_FILESTRING(data.cFileName, RX_FNLEN(data.cFileName));
-    BOOL ret =- ::FindNextFileA(handle, &data);
+    output.emplace_back(FN_TO_RX(x));
+    BOOL ret =- RX_FindNextFile(handle, &data);
     // If the function fails the return value is zero
     // and non-zero otherwise. Not TRUE or FALSE.
     if (ret == FALSE) {
@@ -588,8 +590,7 @@ Status WinEnvIO::GetChildren(const std::string& dir,
 
 Status WinEnvIO::CreateDir(const std::string& name) {
   Status result;
-
+  BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL);
-  BOOL ret = CreateDirectoryA(name.c_str(), NULL);
   if (!ret) {
     auto lastError = GetLastError();
     result = IOErrorFromWindowsError(
@@ -606,7 +607,7 @@ Status  WinEnvIO::CreateDirIfMissing(const std::string& name) {
     return result;
   }
 
-  BOOL ret = CreateDirectoryA(name.c_str(), NULL);
+  BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL);
   if (!ret) {
     auto lastError = GetLastError();
     if (lastError != ERROR_ALREADY_EXISTS) {
@@ -622,7 +623,7 @@ Status  WinEnvIO::CreateDirIfMissing(const std::string& name) {
 
 Status WinEnvIO::DeleteDir(const std::string& name) {
   Status result;
-  BOOL ret = RemoveDirectoryA(name.c_str());
+  BOOL ret = RX_RemoveDirectory(RX_FN(name).c_str());
   if (!ret) {
     auto lastError = GetLastError();
     result = IOErrorFromWindowsError("Failed to remove dir: " + name, lastError);
@@ -635,7 +636,7 @@ Status WinEnvIO::GetFileSize(const std::string& fname,
   Status s;
 
   WIN32_FILE_ATTRIBUTE_DATA attrs;
-  if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) {
+  if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) {
     ULARGE_INTEGER file_size;
     file_size.HighPart = attrs.nFileSizeHigh;
     file_size.LowPart = attrs.nFileSizeLow;
@@ -670,7 +671,7 @@ Status WinEnvIO::GetFileModificationTime(const std::string& fname,
   Status s;
 
   WIN32_FILE_ATTRIBUTE_DATA attrs;
-  if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) {
+  if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) {
     *file_mtime = FileTimeToUnixTime(attrs.ftLastWriteTime);
   } else {
     auto lastError = GetLastError();
@@ -688,7 +689,7 @@ Status WinEnvIO::RenameFile(const std::string& src,
 
   // rename() is not capable of replacing the existing file as on Linux
   // so use OS API directly
-  if (!MoveFileExA(src.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING)) {
+  if (!RX_MoveFileEx(RX_FN(src).c_str(), RX_FN(target).c_str(), MOVEFILE_REPLACE_EXISTING)) {
     DWORD lastError = GetLastError();
 
     std::string text("Failed to rename: ");
@@ -704,7 +705,7 @@ Status WinEnvIO::LinkFile(const std::string& src,
   const std::string& target) {
   Status result;
 
-  if (!CreateHardLinkA(target.c_str(), src.c_str(), NULL)) {
+  if (!RX_CreateHardLink(RX_FN(target).c_str(), RX_FN(src).c_str(),  NULL)) {
     DWORD lastError = GetLastError();
 
     std::string text("Failed to link: ");
@@ -719,7 +720,7 @@ Status WinEnvIO::LinkFile(const std::string& src,
 Status WinEnvIO::NumFileLinks(const std::string& fname, 
   uint64_t* count) {
   Status s;
-  HANDLE handle = ::CreateFileA(fname.c_str(), 0,
+  HANDLE handle = RX_CreateFile(RX_FN(fname).c_str(), 0,
     FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
     NULL,
     OPEN_EXISTING,
@@ -758,7 +759,7 @@ Status WinEnvIO::AreFilesSame(const std::string& first,
   }
 
   // 0 - for access means read metadata
-  HANDLE file_1 = ::CreateFileA(first.c_str(), 0, 
+  HANDLE file_1 = RX_CreateFile(RX_FN(first).c_str(), 0, 
                       FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
                       NULL, 
                       OPEN_EXISTING,
@@ -773,7 +774,7 @@ Status WinEnvIO::AreFilesSame(const std::string& first,
   }
   UniqueCloseHandlePtr g_1(file_1, CloseHandleFunc);
 
-  HANDLE file_2 = ::CreateFileA(second.c_str(), 0,
+  HANDLE file_2 = RX_CreateFile(RX_FN(second).c_str(), 0,
                      FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
                      NULL, OPEN_EXISTING,
                      FILE_FLAG_BACKUP_SEMANTICS, // make opening folders possible
@@ -835,7 +836,7 @@ Status  WinEnvIO::LockFile(const std::string& lockFname,
   HANDLE hFile = 0;
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    hFile = CreateFileA(lockFname.c_str(), (GENERIC_READ | GENERIC_WRITE),
+    hFile = RX_CreateFile(RX_FN(lockFname).c_str(), (GENERIC_READ | GENERIC_WRITE),
       ExclusiveAccessON, NULL, CREATE_ALWAYS,
       FILE_ATTRIBUTE_NORMAL, NULL);
   }
@@ -898,8 +899,8 @@ Status WinEnvIO::NewLogger(const std::string& fname,
   HANDLE hFile = 0;
   {
     IOSTATS_TIMER_GUARD(open_nanos);
-    hFile = CreateFileA(
+    hFile = RX_CreateFile(
-      fname.c_str(), GENERIC_WRITE,
+      RX_FN(fname).c_str(), GENERIC_WRITE,
       FILE_SHARE_READ | FILE_SHARE_DELETE,  // In RocksDb log files are
       // renamed and deleted before
       // they are closed. This enables
@@ -992,17 +993,17 @@ Status WinEnvIO::GetAbsolutePath(const std::string& db_path,
   // For test compatibility we will consider starting slash as an
   // absolute path
   if ((!db_path.empty() && (db_path[0] == '\\' || db_path[0] == '/')) ||
-    !PathIsRelativeA(db_path.c_str())) {
+    !RX_PathIsRelative(RX_FN(db_path).c_str())) {
     *output_path = db_path;
     return Status::OK();
   }
 
-  std::string result;
+  RX_FILESTRING result;
   result.resize(MAX_PATH);
 
   // Hopefully no changes the current directory while we do this
   // however _getcwd also suffers from the same limitation
-  DWORD len = GetCurrentDirectoryA(MAX_PATH, &result[0]);
+  DWORD len = RX_GetCurrentDirectory(MAX_PATH, &result[0]);
   if (len == 0) {
     auto lastError = GetLastError();
     return IOErrorFromWindowsError("Failed to get current working directory",
@@ -1010,8 +1011,9 @@ Status WinEnvIO::GetAbsolutePath(const std::string& db_path,
   }
 
   result.resize(len);
+  std::string res = FN_TO_RX(result);
   
-  result.swap(*output_path);
+  res.swap(*output_path);
   return Status::OK();
 }
 
@@ -1076,7 +1078,7 @@ EnvOptions WinEnvIO::OptimizeForManifestRead(
 // Returns true iff the named directory exists and is a directory.
 bool WinEnvIO::DirExists(const std::string& dname) {
   WIN32_FILE_ATTRIBUTE_DATA attrs;
-  if (GetFileAttributesExA(dname.c_str(), GetFileExInfoStandard, &attrs)) {
+  if (RX_GetFileAttributesEx(RX_FN(dname).c_str(), GetFileExInfoStandard, &attrs)) {
     return 0 != (attrs.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
   }
   return false;
@@ -1085,7 +1087,7 @@ bool WinEnvIO::DirExists(const std::string& dname) {
 size_t WinEnvIO::GetSectorSize(const std::string& fname) {
   size_t sector_size = kSectorSize;
 
-  if (PathIsRelativeA(fname.c_str())) {
+  if (RX_PathIsRelative(RX_FN(fname).c_str())) {
     return sector_size;
   }
 

3rdParty/rocksdb/v5.16.X/port/win/port_win.cc

@@ -26,11 +26,28 @@
 #include <exception>
 #include <chrono>
 
+#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES
+// utf8 <-> utf16
+#include <string>
+#include <locale>
+#include <codecvt>
+#endif
+
 #include "util/logging.h"
 
 namespace rocksdb {
 namespace port {
 
+std::string utf16_to_utf8(const std::wstring& utf16) {
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t> convert;
+  return convert.to_bytes(utf16);
+}
+
+std::wstring utf8_to_utf16(const std::string& utf8) {
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+  return converter.from_bytes(utf8);
+}
+
 void gettimeofday(struct timeval* tv, struct timezone* /* tz */) {
   using namespace std::chrono;
 
@@ -110,7 +127,7 @@ void InitOnce(OnceType* once, void (*initializer)()) {
 struct DIR {
   HANDLE      handle_;
   bool        firstread_;
-  WIN32_FIND_DATA data_;
+  RX_WIN32_FIND_DATA data_;
   dirent entry_;
 
   DIR() : handle_(INVALID_HANDLE_VALUE),
@@ -137,7 +154,7 @@ DIR* opendir(const char* name) {
 
   std::unique_ptr<DIR> dir(new DIR);
 
-  dir->handle_ = ::FindFirstFileExA(pattern.c_str(), 
+  dir->handle_ = RX_FindFirstFileEx(RX_FN(pattern).c_str(), 
     FindExInfoBasic, // Do not want alternative name
     &dir->data_,
     FindExSearchNameMatch,
@@ -148,8 +165,9 @@ DIR* opendir(const char* name) {
     return nullptr;
   }
 
+  RX_FILESTRING x(dir->data_.cFileName, RX_FNLEN(dir->data_.cFileName));
   strcpy_s(dir->entry_.d_name, sizeof(dir->entry_.d_name), 
-    dir->data_.cFileName);
+           FN_TO_RX(x).c_str());
 
   return dir.release();
 }
@@ -165,14 +183,15 @@ struct dirent* readdir(DIR* dirp) {
     return &dirp->entry_;
   }
 
-  auto ret = ::FindNextFileA(dirp->handle_, &dirp->data_);
+  auto ret = RX_FindNextFile(dirp->handle_, &dirp->data_);
 
   if (ret == 0) {
     return nullptr;
   }
 
+  RX_FILESTRING x(dirp->data_.cFileName, RX_FNLEN(dirp->data_.cFileName));
   strcpy_s(dirp->entry_.d_name, sizeof(dirp->entry_.d_name), 
-    dirp->data_.cFileName);
+           FN_TO_RX(x).c_str());
 
   return &dirp->entry_;
 }
@@ -182,11 +201,15 @@ int closedir(DIR* dirp) {
   return 0;
 }
 
-int truncate(const char* path, int64_t len) {
+int truncate(const char* path, int64_t length) {
   if (path == nullptr) {
     errno = EFAULT;
     return -1;
   }
+  return rocksdb::port::Truncate(path, length);
+}
+
+int Truncate(std::string path, int64_t len) {
 
   if (len < 0) {
     errno = EINVAL;
@@ -194,7 +217,7 @@ int truncate(const char* path, int64_t len) {
   }
 
   HANDLE hFile =
-      CreateFile(path, GENERIC_READ | GENERIC_WRITE,
+      RX_CreateFile(RX_FN(path).c_str(), GENERIC_READ | GENERIC_WRITE,
                  FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
                  NULL,           // Security attrs
                  OPEN_EXISTING,  // Truncate existing file only

3rdParty/rocksdb/v5.16.X/port/win/port_win.h

@@ -328,11 +328,62 @@ inline void* pthread_getspecific(pthread_key_t key) {
 // using C-runtime to implement. Note, this does not
 // feel space with zeros in case the file is extended.
 int truncate(const char* path, int64_t length);
+int Truncate(std::string path, int64_t length);
 void Crash(const std::string& srcfile, int srcline);
 extern int GetMaxOpenFiles();
+std::string utf16_to_utf8(const std::wstring& utf16);
+std::wstring utf8_to_utf16(const std::string& utf8);
 
 }  // namespace port
 
+
+#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES
+
+#define RX_FILESTRING std::wstring
+#define RX_FN(a) rocksdb::port::utf8_to_utf16(a)
+#define FN_TO_RX(a) rocksdb::port::utf16_to_utf8(a)
+#define RX_FNLEN(a) ::wcslen(a)
+
+#define RX_DeleteFile DeleteFileW
+#define RX_CreateFile CreateFileW
+#define RX_CreateFileMapping CreateFileMappingW
+#define RX_GetFileAttributesEx GetFileAttributesExW
+#define RX_FindFirstFileEx FindFirstFileExW
+#define RX_FindNextFile FindNextFileW
+#define RX_WIN32_FIND_DATA WIN32_FIND_DATAW
+#define RX_CreateDirectory CreateDirectoryW
+#define RX_RemoveDirectory RemoveDirectoryW
+#define RX_GetFileAttributesEx GetFileAttributesExW
+#define RX_MoveFileEx MoveFileExW
+#define RX_CreateHardLink CreateHardLinkW
+#define RX_PathIsRelative PathIsRelativeW
+#define RX_GetCurrentDirectory GetCurrentDirectoryW
+
+#else
+
+#define RX_FILESTRING std::string
+#define RX_FN(a) a
+#define FN_TO_RX(a) a
+#define RX_FNLEN(a) strlen(a)
+
+#define RX_DeleteFile DeleteFileA
+#define RX_CreateFile CreateFileA
+#define RX_CreateFileMapping CreateFileMappingA
+#define RX_GetFileAttributesEx GetFileAttributesExA
+#define RX_FindFirstFileEx FindFirstFileExA
+#define RX_CreateDirectory CreateDirectoryA
+#define RX_FindNextFile FindNextFileA
+#define RX_WIN32_FIND_DATA WIN32_FIND_DATA
+#define RX_CreateDirectory CreateDirectoryA
+#define RX_RemoveDirectory RemoveDirectoryA
+#define RX_GetFileAttributesEx GetFileAttributesExA
+#define RX_MoveFileEx MoveFileExA
+#define RX_CreateHardLink CreateHardLinkA
+#define RX_PathIsRelative PathIsRelativeA
+#define RX_GetCurrentDirectory GetCurrentDirectoryA
+
+#endif
+
 using port::pthread_key_t;
 using port::pthread_key_create;
 using port::pthread_key_delete;

Installation/Windows/Plugins/Utf8Converter.nsh

@@ -0,0 +1,120 @@
+
+
+/**
+ *  Two macroses to convert ANSI text to UTF-8 and conversely.
+ *  This file is FREE FOR USE.
+ *
+ *  Written by Shurmialiou Vadzim at 04/2010
+ *  inshae@gmail.com
+ */
+ 
+!ifndef ___ANSITOUTF8_NSH___
+!define ___ANSITOUTF8_NSH___
+ 
+!include "LogicLib.nsh"
+ 
+/**
+ * Convert ANSI text to UTF-8 text in the installer or uninstaller.
+ *
+ * Usage:
+ * StrCpy $0 "Belarussian text: Прывiтанне, Свет!"
+ * ${AnsiToUtf8} $0 $1
+ * DetailPrint "'$1' == 'Belarussian text: Прывiтанне, Свет!'"   ;UTF-8 text
+ */
+!define AnsiToUtf8 '!insertmacro AnsiToUtf8Macro'
+!macro AnsiToUtf8Macro SOURCE_STRING OUTPUT_STRING
+	Push "${SOURCE_STRING}"
+	Push 0		;ANSI  codepage
+	Push 65001	;UTF-8 codepage
+	!insertmacro ConvertOverUnicode
+	Pop "${OUTPUT_STRING}"
+!macroend
+ 
+/**
+ * Convert UTF-8 text to ANSI text in the installer or uninstaller.
+ *
+ * Usage:
+ * StrCpy $0 "Belarussian text: Прывiтанне, Свет!"
+ * ${Utf8ToAnsi} $0 $1
+ * DetailPrint "'$1' == 'Belarussian text: Прывiтанне, Свет!'"   ;UTF-8 text
+ */
+!define Utf8ToAnsi '!insertmacro Utf8ToAnsiMacro'
+!macro Utf8ToAnsiMacro SOURCE_STRING OUTPUT_STRING
+	Push "${SOURCE_STRING}"
+	Push 65001	;UTF-8 codepage
+	Push 0		;ANSI  codepage
+	!insertmacro ConvertOverUnicode
+	Pop "${OUTPUT_STRING}"
+!macroend
+ 
+!macro ConvertOverUnicode
+ 
+	Exch $0	;Result codepage
+	Exch
+	Exch $1	;Source codepage
+	Exch
+	Exch 2
+	Exch $2	;Source text
+ 
+	Push $3	;Result text
+	Push $4
+	Push $5	;unicode text
+	Push $6
+	Push $7
+	Push $8
+ 
+	StrCpy $3 ""
+ 
+	;From ANSI to Unicode and then from unicode to UTF-8
+ 
+	${If} $2 != ""
+ 
+		;long bufSize = ::MultiByteToWideChar(CP_ACP, 0, cp1251str, -1, 0, 0);
+		System::Call /NOUNLOAD "kernel32::MultiByteToWideChar(i r1, i 0, t r2, i -1, i 0, i 0) i .r4"
+ 
+		${If} $4 > 0
+ 
+			IntOp $4 $4 * 2	;2 bytes by one unicode-symbol
+			System::Alloc /NOUNLOAD $4
+			Pop $5
+ 
+			; ::MultiByteToWideChar(CP_ACP, 0, cp1251str, -1, unicodeStr, bufSize)
+			System::Call /NOUNLOAD "kernel32::MultiByteToWideChar(i r1, i 0, t r2, i -1, i r5, i r4) i .r6"
+			${If} $6 > 0
+ 
+				;bufSize = ::WideCharToMultiByte(CP_UTF8, 0, unicodeStr, -1, 0, 0, 0, 0);
+				System::Call /NOUNLOAD "kernel32::WideCharToMultiByte(i r0, i 0, i r5, i -1, i 0, i 0, i 0, i 0) i .r6"
+ 
+				; ::WideCharToMultiByte(CP_UTF8, 0, unicodeStr, -1, utf8Str, bufSize, 0, 0)
+				System::Call /NOUNLOAD "kernel32::WideCharToMultiByte(i r0, i 0, i r5, i -1, t .r7, i r6, i 0, i 0) i .r8"
+ 
+				${If} $8 > 0
+					;Save result to $3
+					StrCpy $3 $7
+				${EndIf}
+ 
+			${EndIf}
+ 
+			;Free buffer from unicode string
+			System::Free $5
+ 
+		${EndIf}
+ 
+	${EndIf}
+ 
+	Pop  $8
+	Pop  $7
+	Pop  $6
+	Pop  $5
+	Pop  $4
+	Exch
+	Pop  $2
+	Exch
+	Pop  $1
+	Exch
+	Pop  $0
+	Exch  $3
+ 
+!macroend
+ 
+!endif	;___ANSITOUTF8_NSH___

Installation/Windows/Templates/NSIS.template.in

@@ -15,6 +15,7 @@
   !include "MUI2.nsh"
   !include "StripSlashes.nsh"
   !include "xcopy.nsh"
+  !include "Utf8Converter.nsh"
 ;   !include "GetTime.nsh"
 ;--------------------------------
 ; get commandline parameters
@@ -67,6 +68,7 @@ Var BackupPath
 VAR TRI_INSTALL_SERVICE       ; x bool
 VAR TRI_INSTALL_SCOPE_ALL     ; x bool => All / ThisUser
 Var newCfgValues              ; keep new config file values
+Var newCfgValuesUtf8          ; translated version of the content
 Var newCfgValuesFile          ; write them to a temporary file...
 Var ServiceUp                 ; did the service start?
 !define TEMP1 $R0 ;Temp variable
@@ -207,8 +209,10 @@ Function un.ReadSettings
 
   ;MessageBox MB_OK "Add to path: $ADD_TO_PATH"
   ReadRegStr $ADD_DESKTOP_ICON         SHCTX "${TRI_UNINSTALL_REG_PATH}" "InstallToDesktop"
-  ReadINIStr $DATADIR                  "$INSTDIR\${ARANGO_INI}" "database"   "directory"
+  ReadINIStr $newCfgValuesUtf8                  "$INSTDIR\${ARANGO_INI}" "database"   "directory"
-  ReadINIStr $APPDIR                   "$INSTDIR\${ARANGO_INI}" "javascript" "app-path"
+  ${Utf8ToAnsi} $newCfgValuesUtf8 $DATADIR       
+  ReadINIStr $newCfgValuesUtf8                   "$INSTDIR\${ARANGO_INI}" "javascript" "app-path"
+  ${Utf8ToAnsi} $newCfgValuesUtf8 $APPDIR
 
 FunctionEnd
 
@@ -948,9 +952,10 @@ Section "-Core installation"
   ;  StrCpy $ini_LOGFILE "[log]$\r$\nfile = $LOGFILE$\r$\n"
   ;${EndIf}
   StrCpy $newCfgValues "$ini_APPDIR$ini_DATADIR[server]$\r$\nstorage-engine = $STORAGE_ENGINE$\r$\n"
+  ${AnsiToUtf8} $newCfgValues $newCfgValuesUtf8
   StrCpy $newCfgValuesFile "$INSTDIR\etc\arangodb3\newValues.ini"
   FileOpen $4  "$newCfgValuesFile" w
-  FileWrite $4 "$newCfgValues"
+  FileWrite $4 "$newCfgValuesUtf8"
   FileClose $4
   ; Alter the shipped file and insert the values from above:
   push "$newCfgValuesFile"

arangod/Cluster/ServerState.cpp

@@ -24,8 +24,6 @@
 #include "ServerState.h"
 
 #include <iomanip>
-#include <iostream>
-#include <sstream>
 
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_generators.hpp>
@@ -451,19 +449,24 @@ std::string ServerState::generatePersistedId(RoleEnum const& role) {
 }
 
 std::string ServerState::getPersistedId() {
-  if (hasPersistedId()) {
   std::string uuidFilename = getUuidFilename();
-    std::ifstream ifs(uuidFilename);
+  if (hasPersistedId()) {
-
+    try {
-    std::string id;
+      auto uuidBuf = arangodb::basics::FileUtils::slurp(uuidFilename);
-    if (ifs.is_open()) {
+      basics::StringUtils::trimInPlace(uuidBuf);
-      std::getline(ifs, id);
+      if (!uuidBuf.empty()) {
-      ifs.close();
+        return uuidBuf;
-      return id;
+      }
+    }
+    catch (arangodb::basics::Exception const& ex) {
+      LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Couldn't read UUID file '"
+                                                << uuidFilename << "' - "
+                                                << ex.what();
+      FATAL_ERROR_EXIT();
     }
   }
 
-  LOG_TOPIC(FATAL, Logger::STARTUP) << "Couldn't open UUID file '" << getUuidFilename() << "'";
+  LOG_TOPIC(FATAL, Logger::STARTUP) << "Couldn't open UUID file '" << uuidFilename << "'";
   FATAL_ERROR_EXIT();
 }
 

arangod/RestServer/arangod.cpp

@@ -295,6 +295,7 @@ static void WINAPI ServiceMain(DWORD dwArgc, LPSTR* lpszArgv) {
   // set start pending
   SetServiceStatus(SERVICE_START_PENDING, 0, 1, 10000, 0);
 
+  TRI_GET_ARGV(ARGC, ARGV);
   ArangoGlobalContext context(ARGC, ARGV, SBIN_DIRECTORY);
   runServer(ARGC, ARGV, context);
 
@@ -306,6 +307,7 @@ static void WINAPI ServiceMain(DWORD dwArgc, LPSTR* lpszArgv) {
 #endif
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
 #if _WIN32
   if (argc > 1 && TRI_EqualString("--start-service", argv[1])) {
     ARGC = argc;

arangosh/Benchmark/arangobench.cpp

@@ -45,6 +45,7 @@ using namespace arangodb::application_features;
 using namespace arangodb::basics;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/Dump/arangodump.cpp

@@ -47,6 +47,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/Export/arangoexport.cpp

@@ -44,6 +44,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/Import/arangoimport.cpp

@@ -44,6 +44,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/Restore/arangorestore.cpp

@@ -47,6 +47,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/Shell/ConsoleFeature.cpp

@@ -364,7 +364,7 @@ void ConsoleFeature::print(std::string const& message) {
 
 void ConsoleFeature::openLog() {
   if (!_auditFile.empty()) {
-    _toAuditFile = fopen(_auditFile.c_str(), "w");
+    _toAuditFile = TRI_FOPEN(_auditFile.c_str(), "w");
 
     std::ostringstream s;
 

arangosh/Shell/arangosh.cpp

@@ -49,6 +49,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

arangosh/VPack/arangovpack.cpp

@@ -41,6 +41,7 @@ using namespace arangodb;
 using namespace arangodb::application_features;
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int {
     ArangoGlobalContext context(argc, argv, BIN_DIRECTORY);
     context.installHup();

lib/ApplicationFeatures/DaemonFeature.cpp

@@ -22,12 +22,11 @@
 
 #include "DaemonFeature.h"
 
-#include <fstream>
-#include <iostream>
 #include <thread>
 #include <chrono>
 
 #include "Basics/FileUtils.h"
+#include "Basics/StringUtils.h"
 #include "Logger/LogAppender.h"
 #include "Logger/Logger.h"
 #include "Logger/LoggerFeature.h"
@@ -147,15 +146,30 @@ void DaemonFeature::checkPidFile() {
     } else if (FileUtils::exists(_pidFile) && FileUtils::size(_pidFile) > 0) {
       LOG_TOPIC(INFO, Logger::STARTUP) << "pid-file '" << _pidFile
                                        << "' already exists, verifying pid";
+      std::string oldPidS;
+      try {
+        oldPidS = arangodb::basics::FileUtils::slurp(_pidFile);
+      }
+      catch (arangodb::basics::Exception const& ex) {
+        LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Couldn't read PID file '"
+                                                << _pidFile << "' - "
+                                                << ex.what();
+        FATAL_ERROR_EXIT();
+      }
 
-      std::ifstream f(_pidFile.c_str());
+      basics::StringUtils::trimInPlace(oldPidS);
 
-      // file can be opened
+      if (!oldPidS.empty()) {
-      if (f) {
         TRI_pid_t oldPid;
 
-        f >> oldPid;
+        try {
-
+          oldPid = std::stol(oldPidS);
+        }
+        catch (std::invalid_argument const& ex) {
+          LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "pid-file '" << _pidFile
+                                                    << "' doesn't contain a number.";
+          FATAL_ERROR_EXIT();
+        }
         if (oldPid == 0) {
           LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "pid-file '" << _pidFile
                                                     << "' is unreadable";
@@ -316,15 +330,14 @@ void DaemonFeature::remapStandardFileDescriptors() {
 }
 
 void DaemonFeature::writePidFile(int pid) {
-  std::ofstream out(_pidFile.c_str(), std::ios::trunc);
+  try {
-
+    arangodb::basics::FileUtils::spit(_pidFile, std::to_string(pid), true);
-  if (!out) {
+  }
-    LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "cannot write pid-file '"
+  catch (arangodb::basics::Exception const& ex) {
-                                              << _pidFile << "'";
+      LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "cannot write pid-file '"
-    FATAL_ERROR_EXIT();
+                                                << _pidFile << "' - "
+                                                << ex.what();
   }
-
-  out << pid;
 }
 
 int DaemonFeature::waitForChildProcess(int pid) {

lib/ApplicationFeatures/LanguageFeature.cpp

@@ -70,13 +70,8 @@ void* LanguageFeature::prepareIcu(std::string const& binaryPath,
                                   std::string const& binaryExecutionPath,
                                   std::string& path,
                                   std::string const& binaryName) {
-  char const* icuDataEnv = getenv("ICU_DATA");
   std::string fn("icudtl.dat");
-
+  TRI_GETENV("ICU_DATA", path);
-  if (icuDataEnv != nullptr) {
-    path = FileUtils::buildFilename(icuDataEnv, fn);
-  }
-
   if (path.empty() || !TRI_IsRegularFile(path.c_str())) {
     if (!path.empty()) {
       LOG_TOPIC(WARN, arangodb::Logger::FIXME)
@@ -118,7 +113,8 @@ void* LanguageFeature::prepareIcu(std::string const& binaryPath,
 #ifndef _WIN32
       setenv("ICU_DATA", icu_path.c_str(), 1);
 #else
-      SetEnvironmentVariable("ICU_DATA", icu_path.c_str());
+      UnicodeString uicuEnv(icu_path.c_str(), (uint16_t) icu_path.length());
+      SetEnvironmentVariableW(L"ICU_DATA", uicuEnv.getTerminatedBuffer());
 #endif
     }
   }

lib/Basics/FileUtils.cpp

@@ -30,6 +30,7 @@
 #ifdef TRI_HAVE_DIRECT_H
 #include <direct.h>
 #endif
+#include <unicode/unistr.h>
 
 #include "Basics/Exceptions.h"
 #include "Basics/OpenFilesTracker.h"
@@ -38,15 +39,6 @@
 #include "Basics/tri-strings.h"
 #include "Logger/Logger.h"
 
-#if defined(_WIN32) && defined(_MSC_VER)
-
-#define TRI_DIR_FN(item) item.name
-
-#else
-
-#define TRI_DIR_FN(item) item->d_name
-
-#endif
 
 namespace arangodb {
 namespace basics {
@@ -244,21 +236,6 @@ bool remove(std::string const& fileName, int* errorNumber) {
   return (result != 0) ? false : true;
 }
 
-bool rename(std::string const& oldName, std::string const& newName,
-            int* errorNumber) {
-  if (errorNumber != nullptr) {
-    *errorNumber = 0;
-  }
-
-  int result = std::rename(oldName.c_str(), newName.c_str());
-
-  if (errorNumber != nullptr) {
-    *errorNumber = errno;
-  }
-
-  return (result != 0) ? false : true;
-}
-
 bool createDirectory(std::string const& name, int* errorNumber) {
   if (errorNumber != nullptr) {
     *errorNumber = 0;
@@ -299,17 +276,22 @@ bool copyRecursive(std::string const& source, std::string const& target,
 
 bool copyDirectoryRecursive(std::string const& source,
                             std::string const& target, std::string& error) {
+  char* fn = nullptr;
   bool rc = true;
 
   auto isSubDirectory = [](std::string const& name) -> bool {
     return isDirectory(name);
   };
 #ifdef TRI_HAVE_WIN32_LIST_FILES
-  struct _finddata_t oneItem;
+  struct _wfinddata_t oneItem;
   intptr_t handle;
 
+  std::string rcs;
   std::string filter = source + "\\*";
-  handle = _findfirst(filter.c_str(), &oneItem);
+  
+  UnicodeString f(filter.c_str());
+
+  handle = _wfindfirst(f.getTerminatedBuffer(), &oneItem);
 
   if (handle == -1) {
     error = "directory " + source + "not found";
@@ -317,6 +299,11 @@ bool copyDirectoryRecursive(std::string const& source,
   }
 
   do {
+    rcs.clear();
+    UnicodeString d((wchar_t*) oneItem.name,
+                    static_cast<int32_t>(wcslen(oneItem.name)));
+    d.toUTF8String<std::string>(rcs);
+    fn = (char*) rcs.c_str();
 #else
   DIR* filedir = opendir(source.c_str());
 
@@ -334,16 +321,17 @@ bool copyDirectoryRecursive(std::string const& source,
   // to be thread-safe in reality, and newer versions of POSIX may require its
   // thread-safety formally, and in addition obsolete readdir_r() altogether
   while ((oneItem = (readdir(filedir))) != nullptr) {
+    fn = oneItem->d_name;
 #endif
+
     // Now iterate over the items.
     // check its not the pointer to the upper directory:
-    if (!strcmp(TRI_DIR_FN(oneItem), ".") ||
+    if (!strcmp(fn, ".") ||
-        !strcmp(TRI_DIR_FN(oneItem), "..")) {
+        !strcmp(fn, "..")) {
       continue;
     }
-
+    std::string dst = target + TRI_DIR_SEPARATOR_STR + fn;
-    std::string dst = target + TRI_DIR_SEPARATOR_STR + TRI_DIR_FN(oneItem);
+    std::string src = source + TRI_DIR_SEPARATOR_STR + fn;
-    std::string src = source + TRI_DIR_SEPARATOR_STR + TRI_DIR_FN(oneItem);
 
     // Handle subdirectories:
     if (isSubDirectory(src)) {
@@ -370,7 +358,7 @@ bool copyDirectoryRecursive(std::string const& source,
       }
     }
 #ifdef TRI_HAVE_WIN32_LIST_FILES
-  } while (_findnext(handle, &oneItem) != -1);
+  } while (_wfindnext(handle, &oneItem) != -1);
 
   _findclose(handle);
 
@@ -386,12 +374,15 @@ std::vector<std::string> listFiles(std::string const& directory) {
   std::vector<std::string> result;
 
 #ifdef TRI_HAVE_WIN32_LIST_FILES
+  char* fn = nullptr;
 
-  struct _finddata_t fd;
+  struct _wfinddata_t oneItem;
   intptr_t handle;
+  std::string rcs;
 
   std::string filter = directory + "\\*";
-  handle = _findfirst(filter.c_str(), &fd);
+  UnicodeString f(filter.c_str());
+  handle = _wfindfirst(f.getTerminatedBuffer(), &oneItem);
 
   if (handle == -1) {
     TRI_set_errno(TRI_ERROR_SYS_ERROR);
@@ -402,10 +393,18 @@ std::vector<std::string> listFiles(std::string const& directory) {
   }
 
   do {
-    if (strcmp(fd.name, ".") != 0 && strcmp(fd.name, "..") != 0) {
+    rcs.clear();
-      result.push_back(fd.name);
+    UnicodeString d((wchar_t*) oneItem.name,
+                    static_cast<int32_t>(wcslen(oneItem.name)));
+    d.toUTF8String<std::string>(rcs);
+    fn = (char*) rcs.c_str();
+
+    if (!strcmp(fn, ".") ||
+        !strcmp(fn, "..")) {
+      continue;
     }
-  } while (_findnext(handle, &fd) != -1);
+
+  } while (_wfindnext(handle, &oneItem) != -1);
 
   _findclose(handle);
 
@@ -578,7 +577,8 @@ static void throwProgramError(std::string const& filename) {
 
 std::string slurpProgram(std::string const& program) {
 #ifdef _WIN32
-  FILE* fp = _popen(program.c_str(), "r");
+  UnicodeString uprog(program.c_str(), (uint16_t) program.length());
+  FILE* fp = _wpopen(uprog.getTerminatedBuffer(), L"r");
 #else
   FILE* fp = popen(program.c_str(), "r");
 #endif

lib/Basics/FileUtils.h

@@ -72,10 +72,6 @@ void spit(std::string const& filename, StringBuffer const& content,
 // returns true if a file could be removed
 bool remove(std::string const& fileName, int* errorNumber = nullptr);
 
-// returns true if a file could be renamed
-bool rename(std::string const& oldName, std::string const& newName,
-            int* errorNumber = nullptr);
-
 // creates a new directory
 bool createDirectory(std::string const& name, int* errorNumber = nullptr);
 bool createDirectory(std::string const& name, int mask, int* errorNumber = nullptr);

lib/Basics/files.cpp

@@ -28,6 +28,7 @@
 #include <Shlwapi.h>
 #include <chrono>
 #include <thread>
+#include <unicode/locid.h>
 #endif
 
 #include <algorithm>
@@ -164,14 +165,10 @@ static void ListTreeRecursively(char const* full, char const* path,
 ////////////////////////////////////////////////////////////////////////////////
 
 static std::string LocateConfigDirectoryEnv() {
-  char const* v = getenv("ARANGODB_CONFIG_PATH");
+  std::string r;
-
+  if (!TRI_GETENV("ARANGODB_CONFIG_PATH", r)) {
-  if (v == nullptr) {
     return std::string();
   }
-
-  std::string r(v);
-
   NormalizePath(r);
   while (!r.empty() && IsDirSeparatorChar(r[r.size() - 1])) {
     r.pop_back();
@@ -328,7 +325,8 @@ bool TRI_ExistsFile(char const* path) {
 int TRI_ChMod(char const* path, long mode, std::string& err) {
   int res;
 #ifdef _WIN32
-  res = _chmod(path, static_cast<int>(mode));
+  UnicodeString wpath(path);
+  res = _wchmod(wpath.getTerminatedBuffer(), static_cast<int>(mode));
 #else
   res = chmod(path, mode);
 #endif
@@ -655,18 +653,26 @@ std::vector<std::string> TRI_FilesDirectory(char const* path) {
   std::string filter(path);
   filter.append("\\*");
 
-  struct _finddata_t fd;
+  struct _wfinddata_t fd;
-  intptr_t handle = _findfirst(filter.c_str(), &fd);
+
+  UnicodeString wfilter(filter.c_str());
+  
+  intptr_t handle = _wfindfirst(wfilter.getTerminatedBuffer(), &fd);
 
   if (handle == -1) {
     return result;
   }
 
+  std::string ufn;
+  UnicodeString fn;
   do {
-    if (strcmp(fd.name, ".") != 0 && strcmp(fd.name, "..") != 0) {
+    if (wcscmp(fd.name, L".") != 0 && wcscmp(fd.name, L"..") != 0) {
-      result.emplace_back(fd.name);
+      ufn.clear();
+      fn = fd.name;
+      fn.toUTF8String<std::string>(ufn);
+      result.emplace_back(ufn);
     }
-  } while (_findnext(handle, &fd) != -1);
+  } while (_wfindnext(handle, &fd) != -1);
 
   _findclose(handle);
 
@@ -724,7 +730,11 @@ int TRI_RenameFile(char const* old, char const* filename, long* systemError,
   TRI_ERRORBUF;
 #ifdef _WIN32
   BOOL moveResult = 0;
-  moveResult = MoveFileExA(old, filename,
+
+  UnicodeString oldf(old);
+  UnicodeString newf(filename);
+
+  moveResult = MoveFileExW(oldf.getTerminatedBuffer(), newf.getTerminatedBuffer(),
                            MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING);
 
   if (!moveResult) {
@@ -957,7 +967,8 @@ int TRI_CreateLockFile(char const* filename) {
     }
   }
 
-  HANDLE fd = CreateFile(filename, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
+  UnicodeString fn(filename);
+  HANDLE fd = CreateFileW(fn.getTerminatedBuffer(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS,
                          FILE_ATTRIBUTE_NORMAL, NULL);
 
   if (fd == INVALID_HANDLE_VALUE) {
@@ -1467,13 +1478,13 @@ std::string TRI_BinaryName(char const* argv0) {
 
 std::string TRI_LocateBinaryPath(char const* argv0) {
 #if _WIN32
-  char buff[4096];
+  wchar_t buff[4096];
-  int res = GetModuleFileName(NULL, buff, sizeof(buff));
+  int res = GetModuleFileNameW(NULL, buff, sizeof(buff));
 
   if (res != 0) {
     buff[4095] = '\0';
 
-    char* q = buff + res;
+    wchar_t* q = buff + res;
 
     while (buff < q) {
       if (*q == '\\' || *q == '/') {
@@ -1484,7 +1495,13 @@ std::string TRI_LocateBinaryPath(char const* argv0) {
       --q;
     }
 
-    return std::string(buff);
+    size_t len = q - buff;
+
+    UnicodeString fn(buff, static_cast<int32_t>(len));
+    std::string ufn;
+    fn.toUTF8String<std::string>(ufn);
+    
+    return ufn;
   }
 
   return std::string();
@@ -1506,10 +1523,9 @@ std::string TRI_LocateBinaryPath(char const* argv0) {
 
   // check PATH variable
   else {
-    p = getenv("PATH");
+    std::string pv;
-
+    if (TRI_GETENV("PATH", pv)) {
-    if (p != nullptr) {
+      std::vector<std::string> files = basics::StringUtils::split(pv, ':', '\0');
-      std::vector<std::string> files = basics::StringUtils::split(std::string(p), ':', '\0');
 
       for (auto const& prefix : files) {
         std::string full;
@@ -1659,7 +1675,11 @@ bool TRI_CopyFile(std::string const& src, std::string const& dst,
                   std::string& error) {
 #ifdef _WIN32
   TRI_ERRORBUF;
-  bool rc = CopyFile(src.c_str(), dst.c_str(), true) != 0;
+
+  UnicodeString s(src.c_str());
+  UnicodeString d(dst.c_str());
+
+  bool rc = CopyFileW(s.getTerminatedBuffer(), d.getTerminatedBuffer(), true) != 0;
   if (!rc) {
     TRI_SYSTEM_ERROR();
     error = "failed to copy " + src + " to " + dst + ": " + TRI_GET_ERRORBUF;
@@ -1789,14 +1809,15 @@ bool TRI_CopySymlink(std::string const& srcItem, std::string const& dstItem,
 #ifdef _WIN32
 
 std::string TRI_HomeDirectory() {
-  char const* drive = getenv("HOMEDRIVE");
+  std::string drive;
-  char const* path = getenv("HOMEPATH");
+  std::string path;
 
-  if (drive == nullptr || path == nullptr) {
+  if (! TRI_GETENV("HOMEDRIVE", drive) ||
+      ! TRI_GETENV("HOMEPATH", path)) {
     return std::string();
   }
   
-  return std::string(drive) + std::string(path);
+  return drive + path;
 }
 
 #else
@@ -1833,7 +1854,7 @@ int TRI_Crc32File(char const* path, uint32_t* crc) {
     return TRI_ERROR_OUT_OF_MEMORY;
   }
 
-  fin = fopen(path, "rb");
+  fin = TRI_FOPEN(path, "rb");
 
   if (fin == nullptr) {
     TRI_Free(buffer);
@@ -1901,7 +1922,7 @@ static std::string getTempPath() {
   // ..........................................................................
 
 #define LOCAL_MAX_PATH_BUFFER 2049
-  TCHAR tempPathName[LOCAL_MAX_PATH_BUFFER];
+  wchar_t tempPathName[LOCAL_MAX_PATH_BUFFER];
   DWORD dwReturnValue = 0;
   // ..........................................................................
   // Attempt to locate the path where the users temporary files are stored
@@ -1918,15 +1939,17 @@ static std::string getTempPath() {
      The path specified by the USERPROFILE environment variable.
      The Windows directory.
   */
-  dwReturnValue = GetTempPath(LOCAL_MAX_PATH_BUFFER, tempPathName);
+  dwReturnValue = GetTempPathW(LOCAL_MAX_PATH_BUFFER, tempPathName);
 
   if ((dwReturnValue > LOCAL_MAX_PATH_BUFFER) || (dwReturnValue == 0)) {
     // something wrong
-    LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "GetTempPathA failed: LOCAL_MAX_PATH_BUFFER="
+    LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "GetTempPathW failed: LOCAL_MAX_PATH_BUFFER="
                                               << LOCAL_MAX_PATH_BUFFER << ":dwReturnValue=" << dwReturnValue;
   }
 
-  std::string result(tempPathName);
+  UnicodeString tmpPathW(tempPathName, dwReturnValue);
+  std::string result;
+  tmpPathW.toUTF8String<std::string>(result);
   // ...........................................................................
   // Whether or not UNICODE is defined, we assume that the temporary file name
   // fits in the ascii set of characters. This is a small compromise so that
@@ -1947,8 +1970,14 @@ static std::string getTempPath() {
 }
 
 static int mkDTemp(char* s, size_t bufferSize) {
-  auto rc = _mktemp_s(s, bufferSize);
+  std::string out;
+  UnicodeString sw(s);
+  // this will overwrite the _XXX part of the string:
+  auto rc = _wmktemp_s((wchar_t*)sw.getTerminatedBuffer(), bufferSize);
   if (rc == 0) {
+    // if it worked out, we need to return the utf8 version:
+    sw.toUTF8String<std::string>(out);
+    memcpy(s, out.c_str(), bufferSize);
     rc = TRI_MKDIR(s, 0700);
   }
   return rc;
@@ -2118,7 +2147,7 @@ int TRI_GetTempName(char const* directory, std::string& result, bool createFile,
       errorMessage = std::string("Tempfile already exists! ") + filename;
     } else {
       if (createFile) {
-        FILE* fd = fopen(filename.c_str(), "wb");
+        FILE* fd = TRI_FOPEN(filename.c_str(), "wb");
 
         if (fd != nullptr) {
           fclose(fd);
@@ -2313,7 +2342,8 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) {
 
 bool TRI_PathIsAbsolute(std::string const& path) {
 #if _WIN32
-  return !PathIsRelative(path.c_str());
+  UnicodeString upath(path.c_str(), (uint16_t) path.length());
+  return !PathIsRelativeW(upath.getTerminatedBuffer());
 #else  
   return (!path.empty()) && path.c_str()[0] == '/';
 #endif
@@ -2334,3 +2364,27 @@ void TRI_InitializeFiles() {
 
 void TRI_ShutdownFiles() {}
 
+
+bool TRI_GETENV(char const* which, std::string &value) {
+#ifdef _WIN32
+  UnicodeString uwhich(which);
+  wchar_t const *v = _wgetenv(uwhich.getTerminatedBuffer());
+
+  if (v == nullptr) {
+    return false;
+  }
+  value.clear();
+  UnicodeString vu(v);
+  vu.toUTF8String<std::string>(value);
+  return true;
+#else
+  char const* v = getenv(which);
+
+  if (v == nullptr) {
+    return false;
+  }
+  value.clear();
+  value = v;
+  return true;
+#endif
+}

lib/Basics/files.h

@@ -364,4 +364,9 @@ void TRI_InitializeFiles();
 
 void TRI_ShutdownFiles();
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief if which is found, value is overwriten, true returned.
+////////////////////////////////////////////////////////////////////////////////
+
+bool TRI_GETENV(char const* which, std::string &value);
 #endif

lib/Basics/operating-system.h

@@ -172,6 +172,7 @@
 #define TRI_LSEEK ::lseek
 #define TRI_MKDIR(a, b) ::mkdir((a), (b))
 #define TRI_OPEN(a, b) ::open((a), (b))
+#define TRI_FOPEN(a, b) ::fopen((a), (b))
 #define TRI_READ ::read
 #define TRI_RMDIR ::rmdir
 #define TRI_STAT ::stat
@@ -192,6 +193,7 @@
 #define TRI_LAST_ERROR_STR ::strerror(errno)
 #define TRI_SYSTEM_ERROR() \
   {}
+#define TRI_GET_ARGV(ARGC, ARGV)
 
 // sockets
 
@@ -329,6 +331,7 @@
 #define TRI_LSEEK ::lseek
 #define TRI_MKDIR(a, b) ::mkdir((a), (b))
 #define TRI_OPEN(a, b) ::open((a), (b))
+#define TRI_FOPEN(a, b) ::fopen((a), (b))
 #define TRI_READ ::read
 #define TRI_RMDIR ::rmdir
 #define TRI_STAT ::stat
@@ -349,6 +352,7 @@
 #define TRI_LAST_ERROR_STR ::strerror(errno)
 #define TRI_SYSTEM_ERROR() \
   {}
+#define TRI_GET_ARGV(ARGC, ARGV)
 
 // sockets
 
@@ -473,6 +477,7 @@
 #define TRI_LSEEK ::lseek
 #define TRI_MKDIR(a, b) ::mkdir((a), (b))
 #define TRI_OPEN(a, b) ::open((a), (b))
+#define TRI_FOPEN(a, b) ::fopen((a), (b))
 #define TRI_READ ::read
 #define TRI_RMDIR ::rmdir
 #define TRI_STAT ::stat
@@ -493,6 +498,7 @@
 #define TRI_LAST_ERROR_STR ::strerror(errno)
 #define TRI_SYSTEM_ERROR() \
   {}
+#define TRI_GET_ARGV(ARGC, ARGV)
 
 // sockets
 
@@ -631,6 +637,7 @@
 #define TRI_LSEEK ::lseek
 #define TRI_MKDIR(a, b) ::mkdir((a), (b))
 #define TRI_OPEN(a, b) ::open((a), (b))
+#define TRI_FOPEN(a, b) ::fopen((a), (b))
 #define TRI_READ ::read
 #define TRI_RMDIR ::rmdir
 #define TRI_STAT ::stat
@@ -651,6 +658,7 @@
 #define TRI_LAST_ERROR_STR ::strerror(errno)
 #define TRI_SYSTEM_ERROR() \
   {}
+#define TRI_GET_ARGV(ARGC, ARGV)
 
 // sockets
 
@@ -704,7 +712,6 @@
 #include <WinSock2.h>
 #include <io.h>
 #include <stdio.h>
-
 // available include files
 
 #define TRI_HAVE_DIRECT_H 1
@@ -829,18 +836,13 @@ typedef unsigned char bool;
 
 #define O_RDONLY _O_RDONLY
 
-#define TRI_CHDIR ::_chdir
 #define TRI_CLOSE ::_close
 #define TRI_CREATE(a, b, c) TRI_createFile((a), (b), (c))
 #define TRI_FSTAT ::_fstat64
-#define TRI_GETCWD ::_getcwd
 #define TRI_LSEEK ::_lseeki64
-#define TRI_MKDIR(a, b) ::_mkdir((a))
+#define TRI_MKDIR(a, b) TRI_MKDIR_WIN32(a)
 #define TRI_OPEN(a, b) TRI_OPEN_WIN32((a), (b))
 #define TRI_READ ::_read
-#define TRI_RMDIR ::_rmdir
-#define TRI_STAT ::_stat64
-#define TRI_UNLINK ::_unlink
 #define TRI_WRITE ::_write
 #define TRI_FDOPEN(a, b) ::_fdopen((a), (b))
 
@@ -853,6 +855,17 @@ typedef unsigned char bool;
 
 #define TRI_ERRORBUF char windowsErrorBuf[256] = "";
 #define TRI_GET_ERRORBUF windowsErrorBuf
+#define TRI_GET_ARGV(ARGC, ARGV) TRI_GET_ARGV_WIN(ARGC, ARGV)
+
+// Implemented wrappers in win-utils.cpp:
+FILE* TRI_FOPEN(char const* filename, char const* mode);
+int TRI_CHDIR(char const* dirname);
+int TRI_STAT(char const* path, TRI_stat_t* buffer);
+char* TRI_GETCWD(char* buffer, int maxlen);
+int TRI_MKDIR_WIN32(char const* dirname);
+int TRI_RMDIR(char const* dirname);
+int TRI_UNLINK(char const* filename);
+void TRI_GET_ARGV_WIN(int& argc, char** argv);
 
 // system error string macro requires ERRORBUF to instantiate its buffer before.
 

lib/Basics/process-utils.cpp

@@ -44,6 +44,7 @@
 #ifdef _WIN32
 #include <Psapi.h>
 #include <TlHelp32.h>
+#include <unicode/unistr.h>
 #endif
 
 #include "Logger/Logger.h"
@@ -326,20 +327,57 @@ static int appendQuotedArg(TRI_string_buffer_t* buf, char const* p) {
   return TRI_ERROR_NO_ERROR;
 }
 
-static char* makeWindowsArgs(ExternalProcess* external) {
+
-  TRI_string_buffer_t* buf;
+static int wAppendQuotedArg(std::wstring &buf, wchar_t const* p) {
+  buf += L'"';
+
+  while (*p != 0) {
+    unsigned int i;
+    unsigned int NumberBackslashes = 0;
+    wchar_t const* q = p;
+    while (*q == L'\\') {
+      ++q;
+      ++NumberBackslashes;
+    }
+    if (*q == 0) {
+      // Escape all backslashes, but let the terminating
+      // double quotation mark we add below be interpreted
+      // as a metacharacter.
+      for (i = 0; i < NumberBackslashes; i++) {
+        buf += L'\\';
+        buf += L'\\';
+      }
+      break;
+    } else if (*q == L'"') {
+      // Escape all backslashes and the following
+      // double quotation mark.
+      for (i = 0; i < NumberBackslashes; i++) {
+        buf += L'\\';
+        buf += L'\\';
+      }
+      buf += L'\\';
+      buf += *q;
+    } else {
+      // Backslashes aren't special here.
+      for (i = 0; i < NumberBackslashes; i++) {
+        buf += L'\\';
+      }
+      buf +=  *q;
+    }
+    p = ++q;
+  }
+  buf += L'"';
+  return TRI_ERROR_NO_ERROR;
+}
+static std::wstring makeWindowsArgs(ExternalProcess* external) {
   size_t i;
   int err = TRI_ERROR_NO_ERROR;
-  char* res;
+  std::wstring res;
-
-  buf = TRI_CreateStringBuffer();
-  if (buf == nullptr) {
-    return nullptr;
-  }
 
   if (( external->_executable.find('/') == std::string::npos) && 
       ( external->_executable.find('\\') == std::string::npos)) {
     // oK, this is a binary without path, start the lookup.
+    // This will most probably break with non-ascii paths.
     char buf[MAX_PATH];
     char *pBuf;
     DWORD n;
@@ -349,34 +387,32 @@ static char* makeWindowsArgs(ExternalProcess* external) {
     }
   }
 
-  TRI_ReserveStringBuffer(buf, 1024);
+  UnicodeString uwargs(external->_executable.c_str());
-  err = appendQuotedArg(buf, external->_executable.c_str());
+
+  err = wAppendQuotedArg(res, uwargs.getTerminatedBuffer());
   if (err != TRI_ERROR_NO_ERROR) {
-    TRI_FreeStringBuffer(buf);
     return nullptr;
   }
   for (i = 1; i < external->_numberArguments; i++) {
-    err = TRI_AppendCharStringBuffer(buf, ' ');
+    res += L' ';
+    uwargs = external->_arguments[i];
+    err = wAppendQuotedArg(res, uwargs.getTerminatedBuffer());
     if (err != TRI_ERROR_NO_ERROR) {
-      TRI_FreeStringBuffer(buf);
       return nullptr;
     }
-    err = appendQuotedArg(buf, external->_arguments[i]);
   }
-  res = TRI_StealStringBuffer(buf);
-  TRI_FreeStringBuffer(buf);
   return res;
 }
 
 static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) {
-  char* args;
+  std::wstring args;
   PROCESS_INFORMATION piProcInfo;
-  STARTUPINFO siStartInfo;
+  STARTUPINFOW siStartInfo;
   BOOL bFuncRetn = FALSE;
   TRI_ERRORBUF;
   
   args = makeWindowsArgs(external);
-  if (args == nullptr) {
+  if (args.length() == 0) {
     LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "execute of '" << external->_executable
              << "' failed making args";
     return false;
@@ -386,8 +422,8 @@ static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) {
   ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION));
 
   // set up members of the STARTUPINFO structure
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
+  ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW));
-  siStartInfo.cb = sizeof(STARTUPINFO);
+  siStartInfo.cb = sizeof(STARTUPINFOW);
 
   siStartInfo.dwFlags = STARTF_USESTDHANDLES;
   siStartInfo.hStdInput = rd ? rd : nullptr;
@@ -395,8 +431,8 @@ static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) {
   siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE);
 
   // create the child process
-  bFuncRetn = CreateProcess(NULL,
+  bFuncRetn = CreateProcessW(NULL,
-                            args,  // command line
+                             (LPWSTR)args.c_str(),  // command line
                              NULL,  // process security attributes
                              NULL,  // primary thread security attributes
                              TRUE,  // handles are inherited
@@ -406,8 +442,6 @@ static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) {
                              &siStartInfo,  // STARTUPINFO pointer
                              &piProcInfo);  // receives PROCESS_INFORMATION
 
-  TRI_Free(args);
-
   if (bFuncRetn == FALSE) {
     TRI_SYSTEM_ERROR();
     LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "execute of '" << external->_executable

lib/Basics/tri-zip.cpp

@@ -26,6 +26,7 @@
 #include "Basics/files.h"
 #include "Basics/FileUtils.h"
 #include "Basics/tri-strings.h"
+#include "Basics/Common.h"
 #include "Zip/unzip.h"
 #include "Zip/zip.h"
 
@@ -147,7 +148,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize,
     }
 
     // try to write the outfile
-    fout = fopen(fullPath.c_str(), "wb");
+    fout = TRI_FOPEN(fullPath.c_str(), "wb");
 
     // cannot write to outfile. this may be due to the target directory missing
     if (fout == nullptr && !skipPaths &&
@@ -166,7 +167,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize,
       *(filenameWithoutPath - 1) = c;
 
       // try again
-      fout = fopen(fullPath.c_str(), "wb");
+      fout = TRI_FOPEN(fullPath.c_str(), "wb");
     } else if (fout == nullptr) {
       // try to create the target directory recursively
       // strip filename so we only have the directory name
@@ -178,7 +179,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize,
       }
 
       // try again
-      fout = fopen(fullPath.c_str(), "wb");
+      fout = TRI_FOPEN(fullPath.c_str(), "wb");
     }
 
     if (fout == nullptr) {
@@ -340,7 +341,7 @@ int TRI_ZipFile(char const* filename, char const* dir,
       break;
     }
 
-    FILE* fin = fopen(fullfile.c_str(), "rb");
+    FILE* fin = TRI_FOPEN(fullfile.c_str(), "rb");
 
     if (fin == nullptr) {
       break;
@@ -394,7 +395,7 @@ int TRI_Adler32(char const* filename, uint32_t& checksum) {
   }
   TRI_DEFER(TRI_CLOSE(fd));
 
-  struct TRI_STAT statbuf;
+  TRI_stat_t statbuf;
   int res = TRI_FSTAT(fd, &statbuf);
   if (res < 0) {
     TRI_ERRORBUF;

lib/Basics/win-utils.cpp

@@ -24,6 +24,15 @@
 #include <errno.h>
 
 #include <io.h>
+#include <locale>
+#include <iomanip>
+#include <codecvt>
+#include <sys/stat.h>
+#include <wchar.h>
+#include <unicode/locid.h>
+#include <unicode/uchar.h>
+#include <unicode/unistr.h>
+#include <sys/types.h>
 
 #include "win-utils.h"
 
@@ -33,14 +42,13 @@
 #include <crtdbg.h>
 #include <atlstr.h>
 #include <VersionHelpers.h>
-#include <unicode/uchar.h>
-#include <unicode/unistr.h>
 
 #include "Logger/Logger.h"
 #include "Basics/files.h"
 #include "Basics/StringUtils.h"
 #include "Basics/tri-strings.h"
 #include "Basics/directories.h"
+#include "Basics/Common.h"
 
 using namespace arangodb::basics;
 
@@ -181,11 +189,16 @@ int initializeWindows(const TRI_win_initialize_e initializeWhat,
 int TRI_createFile(char const* filename, int openFlags, int modeFlags) {
   HANDLE fileHandle;
   int fileDescriptor;
+  UnicodeString fn(filename);
   
   fileHandle =
-      CreateFileA(filename, GENERIC_READ | GENERIC_WRITE,
+    CreateFileW(fn.getTerminatedBuffer(),
-                  FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
+                GENERIC_READ | GENERIC_WRITE,
-                  (openFlags & O_APPEND) ? OPEN_ALWAYS : CREATE_NEW, 0, NULL);
+                FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
+                NULL,
+                (openFlags & O_APPEND) ? OPEN_ALWAYS : CREATE_NEW,
+                0,
+                NULL);
 
   if (fileHandle == INVALID_HANDLE_VALUE) {
     return -1;
@@ -228,8 +241,12 @@ int TRI_OPEN_WIN32(char const* filename, int openFlags) {
       break;
   }
 
-  fileHandle = CreateFileA(
+  UnicodeString fn(filename);
-      filename, mode, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
+  fileHandle = CreateFileW(fn.getTerminatedBuffer(),
+                           mode,
+                           FILE_SHARE_DELETE |
+                           FILE_SHARE_READ |
+                           FILE_SHARE_WRITE,
       NULL, OPEN_EXISTING, 0, NULL);
 
   if (fileHandle == INVALID_HANDLE_VALUE) {
@@ -241,6 +258,64 @@ int TRI_OPEN_WIN32(char const* filename, int openFlags) {
   return fileDescriptor;
 }
 
+
+FILE* TRI_FOPEN(char const* filename, char const* mode) {
+  UnicodeString fn(filename);
+  UnicodeString umod(mode);
+  return _wfopen(fn.getTerminatedBuffer(), umod.getTerminatedBuffer());
+}
+
+
+int TRI_CHDIR(char const* dirname) {
+  UnicodeString dn(dirname);
+  return ::_wchdir(dn.getTerminatedBuffer());
+}
+
+int TRI_STAT(char const* path, TRI_stat_t* buffer) {
+  UnicodeString p(path);
+  auto rc =  ::_wstat64(p.getTerminatedBuffer(), buffer);
+  return rc;
+}
+
+char *TRI_GETCWD(char* buffer, int maxlen){
+  char* rc = nullptr;
+  wchar_t* rcw;
+  int wBufLen = maxlen;
+  wchar_t* wbuf = (wchar_t*)malloc(wBufLen * sizeof(wchar_t));
+
+  if (wbuf == nullptr) {
+    return nullptr;
+  }
+  rcw = ::_wgetcwd(wbuf, wBufLen);
+
+  if (rcw != nullptr) {
+    std::string rcs;
+    
+    UnicodeString d(wbuf, static_cast<int32_t>(wcslen(wbuf)));
+    d.toUTF8String<std::string>(rcs);
+    if (rcs.length() + 1 < maxlen) {
+      memcpy(buffer, rcs.c_str(), rcs.length() + 1);
+      rc = buffer;
+    }
+  }
+  free(wbuf);
+  return rc;
+}
+
+int TRI_MKDIR_WIN32(char const* dirname) {
+  UnicodeString dir(dirname);
+  return ::_wmkdir(dir.getTerminatedBuffer());
+}
+  
+int TRI_RMDIR(char const* dirname) {
+  UnicodeString dir(dirname);
+  return ::_wrmdir(dir.getTerminatedBuffer());
+}
+int TRI_UNLINK(char const* filename) {
+  UnicodeString fn(filename);
+  return ::_wunlink(fn.getTerminatedBuffer());
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief converts a Windows error to a *nix system error
 ////////////////////////////////////////////////////////////////////////////////
@@ -411,7 +486,7 @@ int TRI_MapSystemError(DWORD error) {
 static HANDLE hEventLog = INVALID_HANDLE_VALUE;
 
 bool TRI_InitWindowsEventLog(void) {
-  hEventLog = RegisterEventSource(NULL, "ArangoDB");
+  hEventLog = RegisterEventSourceW(NULL, L"ArangoDB");
   if (NULL == hEventLog) {
     // well, fail then.
     return false;
@@ -424,13 +499,6 @@ void TRI_CloseWindowsEventlog(void) {
   hEventLog = INVALID_HANDLE_VALUE;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief logs a message to the windows event log.
-/// we rather are keen on logging something at all then on being able to work
-/// with fancy dynamic buffers; thus we work with a static buffer.
-/// the arango internal logging will handle that usually.
-////////////////////////////////////////////////////////////////////////////////
-
 // No clue why there is no header for these...
 #define MSG_INVALID_COMMAND ((DWORD)0xC0020100L)
 #define UI_CATEGORY ((WORD)0x00000003L)
@@ -447,10 +515,27 @@ void TRI_LogWindowsEventlog(char const* func, char const* file, int line,
   DWORD len = _snprintf(buf, sizeof(buf) - 1, "%s", message.c_str());
   buf[sizeof(buf) - 1] = '\0';
 
+  UnicodeString ubufs[]{
+    UnicodeString(buf, len),
+      UnicodeString(file),
+      UnicodeString(func),
+      UnicodeString(linebuf)
+      };
+  LPCWSTR buffers[] = {
+    ubufs[0].getTerminatedBuffer(),
+    ubufs[1].getTerminatedBuffer(),
+    ubufs[2].getTerminatedBuffer(),
+    ubufs[3].getTerminatedBuffer(), 
+    nullptr
+  };
   // Try to get messages through to windows syslog...
-  if (!ReportEvent(hEventLog, EVENTLOG_ERROR_TYPE, UI_CATEGORY,
+  if (!ReportEventW(hEventLog,
-                   MSG_INVALID_COMMAND, NULL, 4, 0, (LPCSTR*)logBuffers,
+                    EVENTLOG_ERROR_TYPE,
-                   NULL)) {
+                    UI_CATEGORY,
+                    MSG_INVALID_COMMAND,
+                    NULL,
+                    4, 0,
+                    buffers, NULL)) {
     // well, fail then...
   }
 }
@@ -468,10 +553,27 @@ void TRI_LogWindowsEventlog(char const* func, char const* file, int line,
   DWORD len = _vsnprintf(buf, sizeof(buf) - 1, fmt, ap);
   buf[sizeof(buf) - 1] = '\0';
 
+  UnicodeString ubufs[]{
+    UnicodeString(buf, len),
+      UnicodeString(file),
+      UnicodeString(func),
+      UnicodeString(linebuf)
+      };
+  LPCWSTR buffers[] = {
+    ubufs[0].getTerminatedBuffer(),
+    ubufs[1].getTerminatedBuffer(),
+    ubufs[2].getTerminatedBuffer(),
+    ubufs[3].getTerminatedBuffer(), 
+    nullptr
+  };
   // Try to get messages through to windows syslog...
-  if (!ReportEvent(hEventLog, EVENTLOG_ERROR_TYPE, UI_CATEGORY,
+  if (!ReportEventW(hEventLog,
-                   MSG_INVALID_COMMAND, NULL, 4, 0, (LPCSTR*)logBuffers,
+                    EVENTLOG_ERROR_TYPE,
-                   NULL)) {
+                    UI_CATEGORY,
+                    MSG_INVALID_COMMAND,
+                    NULL,
+                    4, 0,
+                    buffers, NULL)) {
     // well, fail then...
   }
 }
@@ -656,3 +758,29 @@ std::string getFileNameFromHandle(HANDLE fileHandle) {
   }
   return std::string((LPCTSTR)CString(FileInformation->FileName));
 }
+
+static std::vector<std::string> argVec;
+
+void TRI_GET_ARGV_WIN(int& argc, char** argv) {
+  auto wargStr = GetCommandLineW();
+
+  // if you want your argc in unicode, all you gonna do 
+  // is ask: 
+  auto wargv = CommandLineToArgvW(wargStr, &argc);
+
+  argVec.reserve(argc);
+
+  UnicodeString buf;
+  std::string uBuf;
+  for (int i = 0; i < argc; i++) {
+    uBuf.clear();
+    // convert one UTF16 argument to utf8:
+    buf = wargv[i];
+    buf.toUTF8String<std::string>(uBuf);
+    // memorize the utf8 value to keep the instance:
+    argVec.push_back(uBuf);
+
+    // Now overwrite our original argc entry with the utf8 one:
+    argv[i] = (char*) argVec[i].c_str();
+  }
+}

lib/Logger/LogAppenderFile.cpp

@@ -202,7 +202,7 @@ void LogAppenderFile::reopenAll() {
     backup.append(".old");
 
     FileUtils::remove(backup);
-    FileUtils::rename(filename, backup);
+    TRI_RenameFile(filename.c_str(), backup.c_str());
 
     // open new log file
     int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(),
@@ -210,7 +210,7 @@ void LogAppenderFile::reopenAll() {
                         S_IRUSR | S_IWUSR | S_IRGRP);
 
     if (fd < 0) {
-      FileUtils::rename(backup, filename);
+      TRI_RenameFile(backup.c_str(), filename.c_str());
       continue;
     }
 

lib/ProgramOptions/IniFileParser.h

@@ -25,10 +25,11 @@
 
 #include "Basics/FileUtils.h"
 
-#include <fstream>
 #include <regex>
+#include <sstream>
 
 #include "ProgramOptions/ProgramOptions.h"
+#include <unicode/unistr.h>
 
 namespace arangodb {
 namespace options {
@@ -67,25 +68,25 @@ class IniFileParser {
       return _options->fail(
           "unable to open configuration file: no configuration file specified");
     }
-
+    std::string buf;
-    std::ifstream ifs(filename, std::ifstream::in);
+    try {
-
+      buf = arangodb::basics::FileUtils::slurp(filename);
-    if (!ifs.is_open()) {
+    }
-      return _options->fail("unable to open configuration file '" + filename +
+    catch (arangodb::basics::Exception const& ex) {
-                            "'");
+      return _options->fail(std::string("Couldn't open configuration file: '") +
+                            filename + "' - " + ex.what());
+      return true;
     }
-
     bool isCommunity = false;
     bool isEnterprise = false;
     std::string currentSection;
     size_t lineNumber = 0;
     
-    while (ifs.good()) {
+    std::istringstream iss(buf);
-      std::string line;
+    for (std::string line; std::getline(iss, line); ) {
+      basics::StringUtils::trimInPlace(line);
       ++lineNumber;
 
-      std::getline(ifs, line);
-
       if (std::regex_match(line, _matchers.comment)) {
         // skip over comments
         continue;

lib/ProgramOptions/Translator.cpp

@@ -76,24 +76,21 @@ std::string arangodb::options::EnvironmentTranslator(std::string const& value,
 
         if (q < e) {
           std::string k = std::string(t, q);
-          char const* v = getenv(k.c_str());
           std::string vv;
-
+          if (!TRI_GETENV(k.c_str(), vv) || (vv.length() == 0)) {
-          if (v != nullptr && *v == '\0') {
-            v = nullptr;
-          }
-
-          if (v == nullptr) {
             auto iter = environment.find(k);
 
             if (iter != environment.end()) {
-              v = iter->second.c_str();
+              vv = iter->second;
             }
           }
 
-          if (v == nullptr) {
+          if (vv.length() == 0) {
+            if (k == "PID") {
+              vv = std::to_string(Thread::currentProcessId());
+            }
 #if _WIN32
-            if (TRI_EqualString(k.c_str(), "ROOTDIR")) {
+            else if (k == "ROOTDIR") {
               vv = TRI_LocateInstallDirectory(nullptr, binaryPath);
 
               if (!vv.empty()) {
@@ -103,13 +100,8 @@ std::string arangodb::options::EnvironmentTranslator(std::string const& value,
                   vv.pop_back();
                 }
               }
-            } else 
-#endif
-            if (TRI_EqualString(k.c_str(), "PID")) {
-              vv = std::to_string(Thread::currentProcessId());
             } 
-          } else {
+#endif
-            vv = v;
           }
 
           result += vv;

lib/V8/v8-utils.cpp

@@ -2695,10 +2695,16 @@ static void JS_Write(v8::FunctionCallbackInfo<v8::Value> const& args) {
   if (args.Length() < 2) {
     TRI_V8_THROW_EXCEPTION_USAGE("write(<filename>, <content>)");
   }
-
+#if _WIN32 // the wintendo needs utf16 filenames
-  TRI_Utf8ValueNFC name(args[0]);
+  v8::String::Value str(args[0]);
-
+  std::wstring name {
-  if (*name == nullptr) {
+    reinterpret_cast<wchar_t *>(*str),
+      static_cast<size_t>(str.length())};
+#else
+  TRI_Utf8ValueNFC str(args[0]);
+  std::string name(*str, str.length());
+#endif
+  if (name.length() == 0) {
     TRI_V8_THROW_TYPE_ERROR("<filename> must be a string");
   }
 
@@ -2722,7 +2728,7 @@ static void JS_Write(v8::FunctionCallbackInfo<v8::Value> const& args) {
     errno = 0;
     // disable exceptions in the stream object:
     file.exceptions(std::ifstream::goodbit);
-    file.open(*name, std::ios::out | std::ios::binary);
+    file.open(name, std::ios::out | std::ios::binary);
 
     if (file.is_open() && file.good()) {
       file.write(data, size);
@@ -2752,7 +2758,7 @@ static void JS_Write(v8::FunctionCallbackInfo<v8::Value> const& args) {
     errno = 0;
     // disable exceptions in the stream object:
     file.exceptions(std::ifstream::goodbit);
-    file.open(*name, std::ios::out | std::ios::binary);
+    file.open(name, std::ios::out | std::ios::binary);
 
     if (file.is_open() && file.good()) {
       file << *content;

lib/Zip/iowin32.cpp

@@ -12,7 +12,7 @@
      For more info read MiniZip_info.txt
 
 */
-
+#include <unicode/unistr.h>
 #include <stdlib.h>
 
 #include "zlib.h"
@@ -108,10 +108,16 @@ win32_open64_file_funcA(voidpf opaque, const void* filename, int mode) {
   win32_translate_open_mode(mode, &dwDesiredAccess, &dwCreationDisposition,
                             &dwShareMode, &dwFlagsAndAttributes);
 
-  if ((filename != NULL) && (dwDesiredAccess != 0))
+  if ((filename != NULL) && (dwDesiredAccess != 0)) {
-    hFile = CreateFileA((LPCSTR)filename, dwDesiredAccess, dwShareMode, NULL,
+    UnicodeString fn((LPSTR)filename);
-                        dwCreationDisposition, dwFlagsAndAttributes, NULL);
+    hFile = CreateFileW(fn.getTerminatedBuffer(),
-
+                        dwDesiredAccess,
+                        dwShareMode,
+                        NULL,
+                        dwCreationDisposition,
+                        dwFlagsAndAttributes,
+                        NULL);
+  }
   return win32_build_iowin(hFile);
 }
 

tests/main.cpp

@@ -50,6 +50,7 @@ private:
 char const* ARGV0 = "";
 
 int main(int argc, char* argv[]) {
+  TRI_GET_ARGV(argc, argv);
   int subargc = 0;
   char **subargv = (char**)malloc(sizeof(char*) * argc);
   bool logLineNumbers = false;