diff --git a/3rdParty/rocksdb/CMakeLists.txt b/3rdParty/rocksdb/CMakeLists.txt index f9cfd46209..df339068aa 100644 --- a/3rdParty/rocksdb/CMakeLists.txt +++ b/3rdParty/rocksdb/CMakeLists.txt @@ -38,6 +38,7 @@ if (MSVC) set(SNAPPY_INCLUDE "${SNAPPY_INCLUDE_DIR}" CACHE PATH "where the wintendo should look for the snappy libs") set(SNAPPY_LIB_DEBUG "${CMAKE_BINARY_DIR}/bin/Debug/${SNAPPY_LIB}.lib") set(SNAPPY_LIB_RELEASE "${CMAKE_BINARY_DIR}/bin/RelWithDebInfo/${SNAPPY_LIB}.lib") + set(WITH_WINDOWS_UTF8_FILENAMES ON CACHE BOOL "we want to provide utf8 filenames") endif () set(USE_RTTI ON CACHE BOOL "enable RTTI") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${ARANGO_ROCKSDB_VERSION} EXCLUDE_FROM_ALL) diff --git a/3rdParty/rocksdb/v5.16.X/CMakeLists.txt b/3rdParty/rocksdb/v5.16.X/CMakeLists.txt index 956a32fc79..456c184ece 100644 --- a/3rdParty/rocksdb/v5.16.X/CMakeLists.txt +++ b/3rdParty/rocksdb/v5.16.X/CMakeLists.txt @@ -49,6 +49,10 @@ option(WITH_SNAPPY "build with SNAPPY" OFF) option(WITH_LZ4 "build with lz4" OFF) option(WITH_ZLIB "build with zlib" OFF) option(WITH_ZSTD "build with zstd" OFF) +option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF) +if (WITH_WINDOWS_UTF8_FILENAMES) + add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES) +endif() if(MSVC) # Defaults currently different for GFLAGS. # We will address find_package work a little later diff --git a/3rdParty/rocksdb/v5.16.X/port/win/env_win.cc b/3rdParty/rocksdb/v5.16.X/port/win/env_win.cc index 043443920e..daa4789dc9 100644 --- a/3rdParty/rocksdb/v5.16.X/port/win/env_win.cc +++ b/3rdParty/rocksdb/v5.16.X/port/win/env_win.cc @@ -102,7 +102,8 @@ WinEnvIO::~WinEnvIO() { Status WinEnvIO::DeleteFile(const std::string& fname) { Status result; - BOOL ret = DeleteFileA(fname.c_str()); + BOOL ret = RX_DeleteFile(RX_FN(fname).c_str()); + if(!ret) { auto lastError = GetLastError(); result = IOErrorFromWindowsError("Failed to delete: " + fname, @@ -114,7 +115,7 @@ Status WinEnvIO::DeleteFile(const std::string& fname) { Status WinEnvIO::Truncate(const std::string& fname, size_t size) { Status s; - int result = truncate(fname.c_str(), size); + int result = rocksdb::port::Truncate(fname, size); if (result != 0) { s = IOError("Failed to truncate: " + fname, errno); } @@ -151,8 +152,8 @@ Status WinEnvIO::NewSequentialFile(const std::string& fname, { IOSTATS_TIMER_GUARD(open_nanos); - hFile = CreateFileA( - fname.c_str(), GENERIC_READ, + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, // Original fopen mode is "rb" fileFlags, NULL); @@ -190,7 +191,7 @@ Status WinEnvIO::NewRandomAccessFile(const std::string& fname, { IOSTATS_TIMER_GUARD(open_nanos); hFile = - CreateFileA(fname.c_str(), GENERIC_READ, + RX_CreateFile(RX_FN(fname).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, fileFlags, NULL); } @@ -217,7 +218,7 @@ Status WinEnvIO::NewRandomAccessFile(const std::string& fname, "NewRandomAccessFile failed to map empty file: " + fname, EINVAL); } - HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READONLY, + HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, // Whole file at its present length 0, NULL); // Mapping name @@ -302,8 +303,8 @@ Status WinEnvIO::OpenWritableFile(const std::string& fname, HANDLE hFile = 0; { IOSTATS_TIMER_GUARD(open_nanos); - hFile = CreateFileA( - fname.c_str(), + hFile = RX_CreateFile( + RX_FN(fname).c_str(), desired_access, // Access desired shared_mode, NULL, // Security attributes @@ -366,7 +367,7 @@ Status WinEnvIO::NewRandomRWFile(const std::string & fname, { IOSTATS_TIMER_GUARD(open_nanos); hFile = - CreateFileA(fname.c_str(), + RX_CreateFile(RX_FN(fname).c_str(), desired_access, shared_mode, NULL, // Security attributes @@ -399,8 +400,8 @@ Status WinEnvIO::NewMemoryMappedFileBuffer(const std::string & fname, HANDLE hFile = INVALID_HANDLE_VALUE; { IOSTATS_TIMER_GUARD(open_nanos); - hFile = CreateFileA( - fname.c_str(), GENERIC_READ | GENERIC_WRITE, + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, // Open only if it exists @@ -432,7 +433,7 @@ Status WinEnvIO::NewMemoryMappedFileBuffer(const std::string & fname, "The specified file size does not fit into 32-bit memory addressing: " + fname); } - HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READWRITE, + HANDLE hMap = RX_CreateFileMapping(hFile, NULL, PAGE_READWRITE, 0, // Whole file at its present length 0, NULL); // Mapping name @@ -483,7 +484,7 @@ Status WinEnvIO::NewDirectory(const std::string& name, // 0 - for access means read metadata { IOSTATS_TIMER_GUARD(open_nanos); - handle = ::CreateFileA(name.c_str(), 0, + handle = RX_CreateFile(RX_FN(name).c_str(), 0, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, @@ -509,8 +510,7 @@ Status WinEnvIO::FileExists(const std::string& fname) { // which is consistent with _access() impl on windows // but can be added WIN32_FILE_ATTRIBUTE_DATA attrs; - if (FALSE == GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, - &attrs)) { + if (FALSE == RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) { auto lastError = GetLastError(); switch (lastError) { case ERROR_ACCESS_DENIED: @@ -535,11 +535,12 @@ Status WinEnvIO::GetChildren(const std::string& dir, result->clear(); std::vector output; - WIN32_FIND_DATA data; + RX_WIN32_FIND_DATA data; + memset(&data, 0, sizeof(data)); std::string pattern(dir); pattern.append("\\").append("*"); - HANDLE handle = ::FindFirstFileExA(pattern.c_str(), + HANDLE handle = RX_FindFirstFileEx(RX_FN(pattern).c_str(), FindExInfoBasic, // Do not want alternative name &data, FindExSearchNameMatch, @@ -572,8 +573,9 @@ Status WinEnvIO::GetChildren(const std::string& dir, data.cFileName[MAX_PATH - 1] = 0; while (true) { - output.emplace_back(data.cFileName); - BOOL ret =- ::FindNextFileA(handle, &data); + auto x = RX_FILESTRING(data.cFileName, RX_FNLEN(data.cFileName)); + output.emplace_back(FN_TO_RX(x)); + BOOL ret =- RX_FindNextFile(handle, &data); // If the function fails the return value is zero // and non-zero otherwise. Not TRUE or FALSE. if (ret == FALSE) { @@ -588,8 +590,7 @@ Status WinEnvIO::GetChildren(const std::string& dir, Status WinEnvIO::CreateDir(const std::string& name) { Status result; - - BOOL ret = CreateDirectoryA(name.c_str(), NULL); + BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL); if (!ret) { auto lastError = GetLastError(); result = IOErrorFromWindowsError( @@ -606,7 +607,7 @@ Status WinEnvIO::CreateDirIfMissing(const std::string& name) { return result; } - BOOL ret = CreateDirectoryA(name.c_str(), NULL); + BOOL ret = RX_CreateDirectory(RX_FN(name).c_str(), NULL); if (!ret) { auto lastError = GetLastError(); if (lastError != ERROR_ALREADY_EXISTS) { @@ -622,7 +623,7 @@ Status WinEnvIO::CreateDirIfMissing(const std::string& name) { Status WinEnvIO::DeleteDir(const std::string& name) { Status result; - BOOL ret = RemoveDirectoryA(name.c_str()); + BOOL ret = RX_RemoveDirectory(RX_FN(name).c_str()); if (!ret) { auto lastError = GetLastError(); result = IOErrorFromWindowsError("Failed to remove dir: " + name, lastError); @@ -635,7 +636,7 @@ Status WinEnvIO::GetFileSize(const std::string& fname, Status s; WIN32_FILE_ATTRIBUTE_DATA attrs; - if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) { + if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) { ULARGE_INTEGER file_size; file_size.HighPart = attrs.nFileSizeHigh; file_size.LowPart = attrs.nFileSizeLow; @@ -670,7 +671,7 @@ Status WinEnvIO::GetFileModificationTime(const std::string& fname, Status s; WIN32_FILE_ATTRIBUTE_DATA attrs; - if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) { + if (RX_GetFileAttributesEx(RX_FN(fname).c_str(), GetFileExInfoStandard, &attrs)) { *file_mtime = FileTimeToUnixTime(attrs.ftLastWriteTime); } else { auto lastError = GetLastError(); @@ -688,7 +689,7 @@ Status WinEnvIO::RenameFile(const std::string& src, // rename() is not capable of replacing the existing file as on Linux // so use OS API directly - if (!MoveFileExA(src.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING)) { + if (!RX_MoveFileEx(RX_FN(src).c_str(), RX_FN(target).c_str(), MOVEFILE_REPLACE_EXISTING)) { DWORD lastError = GetLastError(); std::string text("Failed to rename: "); @@ -704,7 +705,7 @@ Status WinEnvIO::LinkFile(const std::string& src, const std::string& target) { Status result; - if (!CreateHardLinkA(target.c_str(), src.c_str(), NULL)) { + if (!RX_CreateHardLink(RX_FN(target).c_str(), RX_FN(src).c_str(), NULL)) { DWORD lastError = GetLastError(); std::string text("Failed to link: "); @@ -719,7 +720,7 @@ Status WinEnvIO::LinkFile(const std::string& src, Status WinEnvIO::NumFileLinks(const std::string& fname, uint64_t* count) { Status s; - HANDLE handle = ::CreateFileA(fname.c_str(), 0, + HANDLE handle = RX_CreateFile(RX_FN(fname).c_str(), 0, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, @@ -758,7 +759,7 @@ Status WinEnvIO::AreFilesSame(const std::string& first, } // 0 - for access means read metadata - HANDLE file_1 = ::CreateFileA(first.c_str(), 0, + HANDLE file_1 = RX_CreateFile(RX_FN(first).c_str(), 0, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, @@ -773,7 +774,7 @@ Status WinEnvIO::AreFilesSame(const std::string& first, } UniqueCloseHandlePtr g_1(file_1, CloseHandleFunc); - HANDLE file_2 = ::CreateFileA(second.c_str(), 0, + HANDLE file_2 = RX_CreateFile(RX_FN(second).c_str(), 0, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, // make opening folders possible @@ -835,7 +836,7 @@ Status WinEnvIO::LockFile(const std::string& lockFname, HANDLE hFile = 0; { IOSTATS_TIMER_GUARD(open_nanos); - hFile = CreateFileA(lockFname.c_str(), (GENERIC_READ | GENERIC_WRITE), + hFile = RX_CreateFile(RX_FN(lockFname).c_str(), (GENERIC_READ | GENERIC_WRITE), ExclusiveAccessON, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); } @@ -898,8 +899,8 @@ Status WinEnvIO::NewLogger(const std::string& fname, HANDLE hFile = 0; { IOSTATS_TIMER_GUARD(open_nanos); - hFile = CreateFileA( - fname.c_str(), GENERIC_WRITE, + hFile = RX_CreateFile( + RX_FN(fname).c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_DELETE, // In RocksDb log files are // renamed and deleted before // they are closed. This enables @@ -992,17 +993,17 @@ Status WinEnvIO::GetAbsolutePath(const std::string& db_path, // For test compatibility we will consider starting slash as an // absolute path if ((!db_path.empty() && (db_path[0] == '\\' || db_path[0] == '/')) || - !PathIsRelativeA(db_path.c_str())) { + !RX_PathIsRelative(RX_FN(db_path).c_str())) { *output_path = db_path; return Status::OK(); } - std::string result; + RX_FILESTRING result; result.resize(MAX_PATH); // Hopefully no changes the current directory while we do this // however _getcwd also suffers from the same limitation - DWORD len = GetCurrentDirectoryA(MAX_PATH, &result[0]); + DWORD len = RX_GetCurrentDirectory(MAX_PATH, &result[0]); if (len == 0) { auto lastError = GetLastError(); return IOErrorFromWindowsError("Failed to get current working directory", @@ -1010,8 +1011,9 @@ Status WinEnvIO::GetAbsolutePath(const std::string& db_path, } result.resize(len); - - result.swap(*output_path); + std::string res = FN_TO_RX(result); + + res.swap(*output_path); return Status::OK(); } @@ -1076,7 +1078,7 @@ EnvOptions WinEnvIO::OptimizeForManifestRead( // Returns true iff the named directory exists and is a directory. bool WinEnvIO::DirExists(const std::string& dname) { WIN32_FILE_ATTRIBUTE_DATA attrs; - if (GetFileAttributesExA(dname.c_str(), GetFileExInfoStandard, &attrs)) { + if (RX_GetFileAttributesEx(RX_FN(dname).c_str(), GetFileExInfoStandard, &attrs)) { return 0 != (attrs.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY); } return false; @@ -1085,7 +1087,7 @@ bool WinEnvIO::DirExists(const std::string& dname) { size_t WinEnvIO::GetSectorSize(const std::string& fname) { size_t sector_size = kSectorSize; - if (PathIsRelativeA(fname.c_str())) { + if (RX_PathIsRelative(RX_FN(fname).c_str())) { return sector_size; } diff --git a/3rdParty/rocksdb/v5.16.X/port/win/port_win.cc b/3rdParty/rocksdb/v5.16.X/port/win/port_win.cc index 75b4ec6de9..a6587ab01a 100644 --- a/3rdParty/rocksdb/v5.16.X/port/win/port_win.cc +++ b/3rdParty/rocksdb/v5.16.X/port/win/port_win.cc @@ -26,11 +26,28 @@ #include #include +#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES +// utf8 <-> utf16 +#include +#include +#include +#endif + #include "util/logging.h" namespace rocksdb { namespace port { +std::string utf16_to_utf8(const std::wstring& utf16) { + std::wstring_convert,wchar_t> convert; + return convert.to_bytes(utf16); +} + +std::wstring utf8_to_utf16(const std::string& utf8) { + std::wstring_convert> converter; + return converter.from_bytes(utf8); +} + void gettimeofday(struct timeval* tv, struct timezone* /* tz */) { using namespace std::chrono; @@ -110,7 +127,7 @@ void InitOnce(OnceType* once, void (*initializer)()) { struct DIR { HANDLE handle_; bool firstread_; - WIN32_FIND_DATA data_; + RX_WIN32_FIND_DATA data_; dirent entry_; DIR() : handle_(INVALID_HANDLE_VALUE), @@ -137,7 +154,7 @@ DIR* opendir(const char* name) { std::unique_ptr dir(new DIR); - dir->handle_ = ::FindFirstFileExA(pattern.c_str(), + dir->handle_ = RX_FindFirstFileEx(RX_FN(pattern).c_str(), FindExInfoBasic, // Do not want alternative name &dir->data_, FindExSearchNameMatch, @@ -148,8 +165,9 @@ DIR* opendir(const char* name) { return nullptr; } + RX_FILESTRING x(dir->data_.cFileName, RX_FNLEN(dir->data_.cFileName)); strcpy_s(dir->entry_.d_name, sizeof(dir->entry_.d_name), - dir->data_.cFileName); + FN_TO_RX(x).c_str()); return dir.release(); } @@ -165,14 +183,15 @@ struct dirent* readdir(DIR* dirp) { return &dirp->entry_; } - auto ret = ::FindNextFileA(dirp->handle_, &dirp->data_); + auto ret = RX_FindNextFile(dirp->handle_, &dirp->data_); if (ret == 0) { return nullptr; } + RX_FILESTRING x(dirp->data_.cFileName, RX_FNLEN(dirp->data_.cFileName)); strcpy_s(dirp->entry_.d_name, sizeof(dirp->entry_.d_name), - dirp->data_.cFileName); + FN_TO_RX(x).c_str()); return &dirp->entry_; } @@ -182,11 +201,15 @@ int closedir(DIR* dirp) { return 0; } -int truncate(const char* path, int64_t len) { +int truncate(const char* path, int64_t length) { if (path == nullptr) { errno = EFAULT; return -1; } + return rocksdb::port::Truncate(path, length); +} + +int Truncate(std::string path, int64_t len) { if (len < 0) { errno = EINVAL; @@ -194,7 +217,7 @@ int truncate(const char* path, int64_t len) { } HANDLE hFile = - CreateFile(path, GENERIC_READ | GENERIC_WRITE, + RX_CreateFile(RX_FN(path).c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, // Security attrs OPEN_EXISTING, // Truncate existing file only diff --git a/3rdParty/rocksdb/v5.16.X/port/win/port_win.h b/3rdParty/rocksdb/v5.16.X/port/win/port_win.h index 8a6c5aad17..19ce244349 100644 --- a/3rdParty/rocksdb/v5.16.X/port/win/port_win.h +++ b/3rdParty/rocksdb/v5.16.X/port/win/port_win.h @@ -328,11 +328,62 @@ inline void* pthread_getspecific(pthread_key_t key) { // using C-runtime to implement. Note, this does not // feel space with zeros in case the file is extended. int truncate(const char* path, int64_t length); +int Truncate(std::string path, int64_t length); void Crash(const std::string& srcfile, int srcline); extern int GetMaxOpenFiles(); +std::string utf16_to_utf8(const std::wstring& utf16); +std::wstring utf8_to_utf16(const std::string& utf8); } // namespace port + +#ifdef ROCKSDB_WINDOWS_UTF8_FILENAMES + +#define RX_FILESTRING std::wstring +#define RX_FN(a) rocksdb::port::utf8_to_utf16(a) +#define FN_TO_RX(a) rocksdb::port::utf16_to_utf8(a) +#define RX_FNLEN(a) ::wcslen(a) + +#define RX_DeleteFile DeleteFileW +#define RX_CreateFile CreateFileW +#define RX_CreateFileMapping CreateFileMappingW +#define RX_GetFileAttributesEx GetFileAttributesExW +#define RX_FindFirstFileEx FindFirstFileExW +#define RX_FindNextFile FindNextFileW +#define RX_WIN32_FIND_DATA WIN32_FIND_DATAW +#define RX_CreateDirectory CreateDirectoryW +#define RX_RemoveDirectory RemoveDirectoryW +#define RX_GetFileAttributesEx GetFileAttributesExW +#define RX_MoveFileEx MoveFileExW +#define RX_CreateHardLink CreateHardLinkW +#define RX_PathIsRelative PathIsRelativeW +#define RX_GetCurrentDirectory GetCurrentDirectoryW + +#else + +#define RX_FILESTRING std::string +#define RX_FN(a) a +#define FN_TO_RX(a) a +#define RX_FNLEN(a) strlen(a) + +#define RX_DeleteFile DeleteFileA +#define RX_CreateFile CreateFileA +#define RX_CreateFileMapping CreateFileMappingA +#define RX_GetFileAttributesEx GetFileAttributesExA +#define RX_FindFirstFileEx FindFirstFileExA +#define RX_CreateDirectory CreateDirectoryA +#define RX_FindNextFile FindNextFileA +#define RX_WIN32_FIND_DATA WIN32_FIND_DATA +#define RX_CreateDirectory CreateDirectoryA +#define RX_RemoveDirectory RemoveDirectoryA +#define RX_GetFileAttributesEx GetFileAttributesExA +#define RX_MoveFileEx MoveFileExA +#define RX_CreateHardLink CreateHardLinkA +#define RX_PathIsRelative PathIsRelativeA +#define RX_GetCurrentDirectory GetCurrentDirectoryA + +#endif + using port::pthread_key_t; using port::pthread_key_create; using port::pthread_key_delete; diff --git a/Installation/Windows/Plugins/Utf8Converter.nsh b/Installation/Windows/Plugins/Utf8Converter.nsh new file mode 100644 index 0000000000..5e6b76cbb9 --- /dev/null +++ b/Installation/Windows/Plugins/Utf8Converter.nsh @@ -0,0 +1,120 @@ + + +/** + * Two macroses to convert ANSI text to UTF-8 and conversely. + * This file is FREE FOR USE. + * + * Written by Shurmialiou Vadzim at 04/2010 + * inshae@gmail.com + */ + +!ifndef ___ANSITOUTF8_NSH___ +!define ___ANSITOUTF8_NSH___ + +!include "LogicLib.nsh" + +/** + * Convert ANSI text to UTF-8 text in the installer or uninstaller. + * + * Usage: + * StrCpy $0 "Belarussian text: Прывiтанне, Свет!" + * ${AnsiToUtf8} $0 $1 + * DetailPrint "'$1' == 'Belarussian text: Прывiтанне, Свет!'" ;UTF-8 text + */ +!define AnsiToUtf8 '!insertmacro AnsiToUtf8Macro' +!macro AnsiToUtf8Macro SOURCE_STRING OUTPUT_STRING + Push "${SOURCE_STRING}" + Push 0 ;ANSI codepage + Push 65001 ;UTF-8 codepage + !insertmacro ConvertOverUnicode + Pop "${OUTPUT_STRING}" +!macroend + +/** + * Convert UTF-8 text to ANSI text in the installer or uninstaller. + * + * Usage: + * StrCpy $0 "Belarussian text: Прывiтанне, Свет!" + * ${Utf8ToAnsi} $0 $1 + * DetailPrint "'$1' == 'Belarussian text: Прывiтанне, Свет!'" ;UTF-8 text + */ +!define Utf8ToAnsi '!insertmacro Utf8ToAnsiMacro' +!macro Utf8ToAnsiMacro SOURCE_STRING OUTPUT_STRING + Push "${SOURCE_STRING}" + Push 65001 ;UTF-8 codepage + Push 0 ;ANSI codepage + !insertmacro ConvertOverUnicode + Pop "${OUTPUT_STRING}" +!macroend + +!macro ConvertOverUnicode + + Exch $0 ;Result codepage + Exch + Exch $1 ;Source codepage + Exch + Exch 2 + Exch $2 ;Source text + + Push $3 ;Result text + Push $4 + Push $5 ;unicode text + Push $6 + Push $7 + Push $8 + + StrCpy $3 "" + + ;From ANSI to Unicode and then from unicode to UTF-8 + + ${If} $2 != "" + + ;long bufSize = ::MultiByteToWideChar(CP_ACP, 0, cp1251str, -1, 0, 0); + System::Call /NOUNLOAD "kernel32::MultiByteToWideChar(i r1, i 0, t r2, i -1, i 0, i 0) i .r4" + + ${If} $4 > 0 + + IntOp $4 $4 * 2 ;2 bytes by one unicode-symbol + System::Alloc /NOUNLOAD $4 + Pop $5 + + ; ::MultiByteToWideChar(CP_ACP, 0, cp1251str, -1, unicodeStr, bufSize) + System::Call /NOUNLOAD "kernel32::MultiByteToWideChar(i r1, i 0, t r2, i -1, i r5, i r4) i .r6" + ${If} $6 > 0 + + ;bufSize = ::WideCharToMultiByte(CP_UTF8, 0, unicodeStr, -1, 0, 0, 0, 0); + System::Call /NOUNLOAD "kernel32::WideCharToMultiByte(i r0, i 0, i r5, i -1, i 0, i 0, i 0, i 0) i .r6" + + ; ::WideCharToMultiByte(CP_UTF8, 0, unicodeStr, -1, utf8Str, bufSize, 0, 0) + System::Call /NOUNLOAD "kernel32::WideCharToMultiByte(i r0, i 0, i r5, i -1, t .r7, i r6, i 0, i 0) i .r8" + + ${If} $8 > 0 + ;Save result to $3 + StrCpy $3 $7 + ${EndIf} + + ${EndIf} + + ;Free buffer from unicode string + System::Free $5 + + ${EndIf} + + ${EndIf} + + Pop $8 + Pop $7 + Pop $6 + Pop $5 + Pop $4 + Exch + Pop $2 + Exch + Pop $1 + Exch + Pop $0 + Exch $3 + +!macroend + +!endif ;___ANSITOUTF8_NSH___ diff --git a/Installation/Windows/Templates/NSIS.template.in b/Installation/Windows/Templates/NSIS.template.in index 4d7c7fe41d..6c1f20c120 100755 --- a/Installation/Windows/Templates/NSIS.template.in +++ b/Installation/Windows/Templates/NSIS.template.in @@ -15,6 +15,7 @@ !include "MUI2.nsh" !include "StripSlashes.nsh" !include "xcopy.nsh" + !include "Utf8Converter.nsh" ; !include "GetTime.nsh" ;-------------------------------- ; get commandline parameters @@ -67,6 +68,7 @@ Var BackupPath VAR TRI_INSTALL_SERVICE ; x bool VAR TRI_INSTALL_SCOPE_ALL ; x bool => All / ThisUser Var newCfgValues ; keep new config file values +Var newCfgValuesUtf8 ; translated version of the content Var newCfgValuesFile ; write them to a temporary file... Var ServiceUp ; did the service start? !define TEMP1 $R0 ;Temp variable @@ -207,8 +209,10 @@ Function un.ReadSettings ;MessageBox MB_OK "Add to path: $ADD_TO_PATH" ReadRegStr $ADD_DESKTOP_ICON SHCTX "${TRI_UNINSTALL_REG_PATH}" "InstallToDesktop" - ReadINIStr $DATADIR "$INSTDIR\${ARANGO_INI}" "database" "directory" - ReadINIStr $APPDIR "$INSTDIR\${ARANGO_INI}" "javascript" "app-path" + ReadINIStr $newCfgValuesUtf8 "$INSTDIR\${ARANGO_INI}" "database" "directory" + ${Utf8ToAnsi} $newCfgValuesUtf8 $DATADIR + ReadINIStr $newCfgValuesUtf8 "$INSTDIR\${ARANGO_INI}" "javascript" "app-path" + ${Utf8ToAnsi} $newCfgValuesUtf8 $APPDIR FunctionEnd @@ -948,9 +952,10 @@ Section "-Core installation" ; StrCpy $ini_LOGFILE "[log]$\r$\nfile = $LOGFILE$\r$\n" ;${EndIf} StrCpy $newCfgValues "$ini_APPDIR$ini_DATADIR[server]$\r$\nstorage-engine = $STORAGE_ENGINE$\r$\n" + ${AnsiToUtf8} $newCfgValues $newCfgValuesUtf8 StrCpy $newCfgValuesFile "$INSTDIR\etc\arangodb3\newValues.ini" FileOpen $4 "$newCfgValuesFile" w - FileWrite $4 "$newCfgValues" + FileWrite $4 "$newCfgValuesUtf8" FileClose $4 ; Alter the shipped file and insert the values from above: push "$newCfgValuesFile" diff --git a/arangod/Cluster/ServerState.cpp b/arangod/Cluster/ServerState.cpp index 40235a866e..daa8ef2dc2 100644 --- a/arangod/Cluster/ServerState.cpp +++ b/arangod/Cluster/ServerState.cpp @@ -24,8 +24,6 @@ #include "ServerState.h" #include -#include -#include #include #include @@ -451,19 +449,24 @@ std::string ServerState::generatePersistedId(RoleEnum const& role) { } std::string ServerState::getPersistedId() { + std::string uuidFilename = getUuidFilename(); if (hasPersistedId()) { - std::string uuidFilename = getUuidFilename(); - std::ifstream ifs(uuidFilename); - - std::string id; - if (ifs.is_open()) { - std::getline(ifs, id); - ifs.close(); - return id; + try { + auto uuidBuf = arangodb::basics::FileUtils::slurp(uuidFilename); + basics::StringUtils::trimInPlace(uuidBuf); + if (!uuidBuf.empty()) { + return uuidBuf; + } + } + catch (arangodb::basics::Exception const& ex) { + LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Couldn't read UUID file '" + << uuidFilename << "' - " + << ex.what(); + FATAL_ERROR_EXIT(); } } - LOG_TOPIC(FATAL, Logger::STARTUP) << "Couldn't open UUID file '" << getUuidFilename() << "'"; + LOG_TOPIC(FATAL, Logger::STARTUP) << "Couldn't open UUID file '" << uuidFilename << "'"; FATAL_ERROR_EXIT(); } diff --git a/arangod/RestServer/arangod.cpp b/arangod/RestServer/arangod.cpp index 0dc11cbeda..6f58f57d2b 100644 --- a/arangod/RestServer/arangod.cpp +++ b/arangod/RestServer/arangod.cpp @@ -295,6 +295,7 @@ static void WINAPI ServiceMain(DWORD dwArgc, LPSTR* lpszArgv) { // set start pending SetServiceStatus(SERVICE_START_PENDING, 0, 1, 10000, 0); + TRI_GET_ARGV(ARGC, ARGV); ArangoGlobalContext context(ARGC, ARGV, SBIN_DIRECTORY); runServer(ARGC, ARGV, context); @@ -306,6 +307,7 @@ static void WINAPI ServiceMain(DWORD dwArgc, LPSTR* lpszArgv) { #endif int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); #if _WIN32 if (argc > 1 && TRI_EqualString("--start-service", argv[1])) { ARGC = argc; diff --git a/arangosh/Benchmark/arangobench.cpp b/arangosh/Benchmark/arangobench.cpp index 8c6ba1a50d..8dd0446f5b 100644 --- a/arangosh/Benchmark/arangobench.cpp +++ b/arangosh/Benchmark/arangobench.cpp @@ -45,6 +45,7 @@ using namespace arangodb::application_features; using namespace arangodb::basics; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/Dump/arangodump.cpp b/arangosh/Dump/arangodump.cpp index 40d16be7a2..82cdbe2517 100644 --- a/arangosh/Dump/arangodump.cpp +++ b/arangosh/Dump/arangodump.cpp @@ -47,6 +47,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/Export/arangoexport.cpp b/arangosh/Export/arangoexport.cpp index b414d41b04..5dc738b080 100644 --- a/arangosh/Export/arangoexport.cpp +++ b/arangosh/Export/arangoexport.cpp @@ -44,6 +44,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/Import/arangoimport.cpp b/arangosh/Import/arangoimport.cpp index 306a138c58..78530ced74 100644 --- a/arangosh/Import/arangoimport.cpp +++ b/arangosh/Import/arangoimport.cpp @@ -44,6 +44,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/Restore/arangorestore.cpp b/arangosh/Restore/arangorestore.cpp index 3ae4247696..ea104afafe 100644 --- a/arangosh/Restore/arangorestore.cpp +++ b/arangosh/Restore/arangorestore.cpp @@ -47,6 +47,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/Shell/ConsoleFeature.cpp b/arangosh/Shell/ConsoleFeature.cpp index 333b537ffe..5ef8e6a7e7 100644 --- a/arangosh/Shell/ConsoleFeature.cpp +++ b/arangosh/Shell/ConsoleFeature.cpp @@ -364,7 +364,7 @@ void ConsoleFeature::print(std::string const& message) { void ConsoleFeature::openLog() { if (!_auditFile.empty()) { - _toAuditFile = fopen(_auditFile.c_str(), "w"); + _toAuditFile = TRI_FOPEN(_auditFile.c_str(), "w"); std::ostringstream s; diff --git a/arangosh/Shell/arangosh.cpp b/arangosh/Shell/arangosh.cpp index 76ac93311a..3c293cc343 100644 --- a/arangosh/Shell/arangosh.cpp +++ b/arangosh/Shell/arangosh.cpp @@ -49,6 +49,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/arangosh/VPack/arangovpack.cpp b/arangosh/VPack/arangovpack.cpp index b472246e0f..577fea7a41 100644 --- a/arangosh/VPack/arangovpack.cpp +++ b/arangosh/VPack/arangovpack.cpp @@ -41,6 +41,7 @@ using namespace arangodb; using namespace arangodb::application_features; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); return ClientFeature::runMain(argc, argv, [&](int argc, char* argv[]) -> int { ArangoGlobalContext context(argc, argv, BIN_DIRECTORY); context.installHup(); diff --git a/lib/ApplicationFeatures/DaemonFeature.cpp b/lib/ApplicationFeatures/DaemonFeature.cpp index 82b20a1420..52ef8a51b6 100644 --- a/lib/ApplicationFeatures/DaemonFeature.cpp +++ b/lib/ApplicationFeatures/DaemonFeature.cpp @@ -22,12 +22,11 @@ #include "DaemonFeature.h" -#include -#include #include #include #include "Basics/FileUtils.h" +#include "Basics/StringUtils.h" #include "Logger/LogAppender.h" #include "Logger/Logger.h" #include "Logger/LoggerFeature.h" @@ -147,15 +146,30 @@ void DaemonFeature::checkPidFile() { } else if (FileUtils::exists(_pidFile) && FileUtils::size(_pidFile) > 0) { LOG_TOPIC(INFO, Logger::STARTUP) << "pid-file '" << _pidFile << "' already exists, verifying pid"; + std::string oldPidS; + try { + oldPidS = arangodb::basics::FileUtils::slurp(_pidFile); + } + catch (arangodb::basics::Exception const& ex) { + LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "Couldn't read PID file '" + << _pidFile << "' - " + << ex.what(); + FATAL_ERROR_EXIT(); + } - std::ifstream f(_pidFile.c_str()); + basics::StringUtils::trimInPlace(oldPidS); - // file can be opened - if (f) { + if (!oldPidS.empty()) { TRI_pid_t oldPid; - f >> oldPid; - + try { + oldPid = std::stol(oldPidS); + } + catch (std::invalid_argument const& ex) { + LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "pid-file '" << _pidFile + << "' doesn't contain a number."; + FATAL_ERROR_EXIT(); + } if (oldPid == 0) { LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "pid-file '" << _pidFile << "' is unreadable"; @@ -316,15 +330,14 @@ void DaemonFeature::remapStandardFileDescriptors() { } void DaemonFeature::writePidFile(int pid) { - std::ofstream out(_pidFile.c_str(), std::ios::trunc); - - if (!out) { - LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "cannot write pid-file '" - << _pidFile << "'"; - FATAL_ERROR_EXIT(); + try { + arangodb::basics::FileUtils::spit(_pidFile, std::to_string(pid), true); + } + catch (arangodb::basics::Exception const& ex) { + LOG_TOPIC(FATAL, arangodb::Logger::FIXME) << "cannot write pid-file '" + << _pidFile << "' - " + << ex.what(); } - - out << pid; } int DaemonFeature::waitForChildProcess(int pid) { diff --git a/lib/ApplicationFeatures/LanguageFeature.cpp b/lib/ApplicationFeatures/LanguageFeature.cpp index a8ca18e546..66096ffe5f 100644 --- a/lib/ApplicationFeatures/LanguageFeature.cpp +++ b/lib/ApplicationFeatures/LanguageFeature.cpp @@ -70,13 +70,8 @@ void* LanguageFeature::prepareIcu(std::string const& binaryPath, std::string const& binaryExecutionPath, std::string& path, std::string const& binaryName) { - char const* icuDataEnv = getenv("ICU_DATA"); std::string fn("icudtl.dat"); - - if (icuDataEnv != nullptr) { - path = FileUtils::buildFilename(icuDataEnv, fn); - } - + TRI_GETENV("ICU_DATA", path); if (path.empty() || !TRI_IsRegularFile(path.c_str())) { if (!path.empty()) { LOG_TOPIC(WARN, arangodb::Logger::FIXME) @@ -118,7 +113,8 @@ void* LanguageFeature::prepareIcu(std::string const& binaryPath, #ifndef _WIN32 setenv("ICU_DATA", icu_path.c_str(), 1); #else - SetEnvironmentVariable("ICU_DATA", icu_path.c_str()); + UnicodeString uicuEnv(icu_path.c_str(), (uint16_t) icu_path.length()); + SetEnvironmentVariableW(L"ICU_DATA", uicuEnv.getTerminatedBuffer()); #endif } } diff --git a/lib/Basics/FileUtils.cpp b/lib/Basics/FileUtils.cpp index 0972da987c..8b2d36c444 100644 --- a/lib/Basics/FileUtils.cpp +++ b/lib/Basics/FileUtils.cpp @@ -30,6 +30,7 @@ #ifdef TRI_HAVE_DIRECT_H #include #endif +#include #include "Basics/Exceptions.h" #include "Basics/OpenFilesTracker.h" @@ -38,15 +39,6 @@ #include "Basics/tri-strings.h" #include "Logger/Logger.h" -#if defined(_WIN32) && defined(_MSC_VER) - -#define TRI_DIR_FN(item) item.name - -#else - -#define TRI_DIR_FN(item) item->d_name - -#endif namespace arangodb { namespace basics { @@ -244,21 +236,6 @@ bool remove(std::string const& fileName, int* errorNumber) { return (result != 0) ? false : true; } -bool rename(std::string const& oldName, std::string const& newName, - int* errorNumber) { - if (errorNumber != nullptr) { - *errorNumber = 0; - } - - int result = std::rename(oldName.c_str(), newName.c_str()); - - if (errorNumber != nullptr) { - *errorNumber = errno; - } - - return (result != 0) ? false : true; -} - bool createDirectory(std::string const& name, int* errorNumber) { if (errorNumber != nullptr) { *errorNumber = 0; @@ -299,17 +276,22 @@ bool copyRecursive(std::string const& source, std::string const& target, bool copyDirectoryRecursive(std::string const& source, std::string const& target, std::string& error) { + char* fn = nullptr; bool rc = true; auto isSubDirectory = [](std::string const& name) -> bool { return isDirectory(name); }; #ifdef TRI_HAVE_WIN32_LIST_FILES - struct _finddata_t oneItem; + struct _wfinddata_t oneItem; intptr_t handle; + std::string rcs; std::string filter = source + "\\*"; - handle = _findfirst(filter.c_str(), &oneItem); + + UnicodeString f(filter.c_str()); + + handle = _wfindfirst(f.getTerminatedBuffer(), &oneItem); if (handle == -1) { error = "directory " + source + "not found"; @@ -317,6 +299,11 @@ bool copyDirectoryRecursive(std::string const& source, } do { + rcs.clear(); + UnicodeString d((wchar_t*) oneItem.name, + static_cast(wcslen(oneItem.name))); + d.toUTF8String(rcs); + fn = (char*) rcs.c_str(); #else DIR* filedir = opendir(source.c_str()); @@ -334,16 +321,17 @@ bool copyDirectoryRecursive(std::string const& source, // to be thread-safe in reality, and newer versions of POSIX may require its // thread-safety formally, and in addition obsolete readdir_r() altogether while ((oneItem = (readdir(filedir))) != nullptr) { + fn = oneItem->d_name; #endif + // Now iterate over the items. // check its not the pointer to the upper directory: - if (!strcmp(TRI_DIR_FN(oneItem), ".") || - !strcmp(TRI_DIR_FN(oneItem), "..")) { + if (!strcmp(fn, ".") || + !strcmp(fn, "..")) { continue; } - - std::string dst = target + TRI_DIR_SEPARATOR_STR + TRI_DIR_FN(oneItem); - std::string src = source + TRI_DIR_SEPARATOR_STR + TRI_DIR_FN(oneItem); + std::string dst = target + TRI_DIR_SEPARATOR_STR + fn; + std::string src = source + TRI_DIR_SEPARATOR_STR + fn; // Handle subdirectories: if (isSubDirectory(src)) { @@ -370,7 +358,7 @@ bool copyDirectoryRecursive(std::string const& source, } } #ifdef TRI_HAVE_WIN32_LIST_FILES - } while (_findnext(handle, &oneItem) != -1); + } while (_wfindnext(handle, &oneItem) != -1); _findclose(handle); @@ -386,12 +374,15 @@ std::vector listFiles(std::string const& directory) { std::vector result; #ifdef TRI_HAVE_WIN32_LIST_FILES + char* fn = nullptr; - struct _finddata_t fd; + struct _wfinddata_t oneItem; intptr_t handle; + std::string rcs; std::string filter = directory + "\\*"; - handle = _findfirst(filter.c_str(), &fd); + UnicodeString f(filter.c_str()); + handle = _wfindfirst(f.getTerminatedBuffer(), &oneItem); if (handle == -1) { TRI_set_errno(TRI_ERROR_SYS_ERROR); @@ -402,10 +393,18 @@ std::vector listFiles(std::string const& directory) { } do { - if (strcmp(fd.name, ".") != 0 && strcmp(fd.name, "..") != 0) { - result.push_back(fd.name); + rcs.clear(); + UnicodeString d((wchar_t*) oneItem.name, + static_cast(wcslen(oneItem.name))); + d.toUTF8String(rcs); + fn = (char*) rcs.c_str(); + + if (!strcmp(fn, ".") || + !strcmp(fn, "..")) { + continue; } - } while (_findnext(handle, &fd) != -1); + + } while (_wfindnext(handle, &oneItem) != -1); _findclose(handle); @@ -578,7 +577,8 @@ static void throwProgramError(std::string const& filename) { std::string slurpProgram(std::string const& program) { #ifdef _WIN32 - FILE* fp = _popen(program.c_str(), "r"); + UnicodeString uprog(program.c_str(), (uint16_t) program.length()); + FILE* fp = _wpopen(uprog.getTerminatedBuffer(), L"r"); #else FILE* fp = popen(program.c_str(), "r"); #endif diff --git a/lib/Basics/FileUtils.h b/lib/Basics/FileUtils.h index ce8d0485f8..8450a9b3c1 100644 --- a/lib/Basics/FileUtils.h +++ b/lib/Basics/FileUtils.h @@ -72,10 +72,6 @@ void spit(std::string const& filename, StringBuffer const& content, // returns true if a file could be removed bool remove(std::string const& fileName, int* errorNumber = nullptr); -// returns true if a file could be renamed -bool rename(std::string const& oldName, std::string const& newName, - int* errorNumber = nullptr); - // creates a new directory bool createDirectory(std::string const& name, int* errorNumber = nullptr); bool createDirectory(std::string const& name, int mask, int* errorNumber = nullptr); diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index f994f9a4a4..4a2662f05c 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #endif #include @@ -164,14 +165,10 @@ static void ListTreeRecursively(char const* full, char const* path, //////////////////////////////////////////////////////////////////////////////// static std::string LocateConfigDirectoryEnv() { - char const* v = getenv("ARANGODB_CONFIG_PATH"); - - if (v == nullptr) { + std::string r; + if (!TRI_GETENV("ARANGODB_CONFIG_PATH", r)) { return std::string(); } - - std::string r(v); - NormalizePath(r); while (!r.empty() && IsDirSeparatorChar(r[r.size() - 1])) { r.pop_back(); @@ -328,7 +325,8 @@ bool TRI_ExistsFile(char const* path) { int TRI_ChMod(char const* path, long mode, std::string& err) { int res; #ifdef _WIN32 - res = _chmod(path, static_cast(mode)); + UnicodeString wpath(path); + res = _wchmod(wpath.getTerminatedBuffer(), static_cast(mode)); #else res = chmod(path, mode); #endif @@ -655,18 +653,26 @@ std::vector TRI_FilesDirectory(char const* path) { std::string filter(path); filter.append("\\*"); - struct _finddata_t fd; - intptr_t handle = _findfirst(filter.c_str(), &fd); + struct _wfinddata_t fd; + + UnicodeString wfilter(filter.c_str()); + + intptr_t handle = _wfindfirst(wfilter.getTerminatedBuffer(), &fd); if (handle == -1) { return result; } + std::string ufn; + UnicodeString fn; do { - if (strcmp(fd.name, ".") != 0 && strcmp(fd.name, "..") != 0) { - result.emplace_back(fd.name); + if (wcscmp(fd.name, L".") != 0 && wcscmp(fd.name, L"..") != 0) { + ufn.clear(); + fn = fd.name; + fn.toUTF8String(ufn); + result.emplace_back(ufn); } - } while (_findnext(handle, &fd) != -1); + } while (_wfindnext(handle, &fd) != -1); _findclose(handle); @@ -724,7 +730,11 @@ int TRI_RenameFile(char const* old, char const* filename, long* systemError, TRI_ERRORBUF; #ifdef _WIN32 BOOL moveResult = 0; - moveResult = MoveFileExA(old, filename, + + UnicodeString oldf(old); + UnicodeString newf(filename); + + moveResult = MoveFileExW(oldf.getTerminatedBuffer(), newf.getTerminatedBuffer(), MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING); if (!moveResult) { @@ -956,8 +966,9 @@ int TRI_CreateLockFile(char const* filename) { return TRI_ERROR_NO_ERROR; } } - - HANDLE fd = CreateFile(filename, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, + + UnicodeString fn(filename); + HANDLE fd = CreateFileW(fn.getTerminatedBuffer(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); if (fd == INVALID_HANDLE_VALUE) { @@ -1467,13 +1478,13 @@ std::string TRI_BinaryName(char const* argv0) { std::string TRI_LocateBinaryPath(char const* argv0) { #if _WIN32 - char buff[4096]; - int res = GetModuleFileName(NULL, buff, sizeof(buff)); + wchar_t buff[4096]; + int res = GetModuleFileNameW(NULL, buff, sizeof(buff)); if (res != 0) { buff[4095] = '\0'; - char* q = buff + res; + wchar_t* q = buff + res; while (buff < q) { if (*q == '\\' || *q == '/') { @@ -1484,7 +1495,13 @@ std::string TRI_LocateBinaryPath(char const* argv0) { --q; } - return std::string(buff); + size_t len = q - buff; + + UnicodeString fn(buff, static_cast(len)); + std::string ufn; + fn.toUTF8String(ufn); + + return ufn; } return std::string(); @@ -1506,10 +1523,9 @@ std::string TRI_LocateBinaryPath(char const* argv0) { // check PATH variable else { - p = getenv("PATH"); - - if (p != nullptr) { - std::vector files = basics::StringUtils::split(std::string(p), ':', '\0'); + std::string pv; + if (TRI_GETENV("PATH", pv)) { + std::vector files = basics::StringUtils::split(pv, ':', '\0'); for (auto const& prefix : files) { std::string full; @@ -1659,7 +1675,11 @@ bool TRI_CopyFile(std::string const& src, std::string const& dst, std::string& error) { #ifdef _WIN32 TRI_ERRORBUF; - bool rc = CopyFile(src.c_str(), dst.c_str(), true) != 0; + + UnicodeString s(src.c_str()); + UnicodeString d(dst.c_str()); + + bool rc = CopyFileW(s.getTerminatedBuffer(), d.getTerminatedBuffer(), true) != 0; if (!rc) { TRI_SYSTEM_ERROR(); error = "failed to copy " + src + " to " + dst + ": " + TRI_GET_ERRORBUF; @@ -1789,14 +1809,15 @@ bool TRI_CopySymlink(std::string const& srcItem, std::string const& dstItem, #ifdef _WIN32 std::string TRI_HomeDirectory() { - char const* drive = getenv("HOMEDRIVE"); - char const* path = getenv("HOMEPATH"); + std::string drive; + std::string path; - if (drive == nullptr || path == nullptr) { + if (! TRI_GETENV("HOMEDRIVE", drive) || + ! TRI_GETENV("HOMEPATH", path)) { return std::string(); } - return std::string(drive) + std::string(path); + return drive + path; } #else @@ -1833,7 +1854,7 @@ int TRI_Crc32File(char const* path, uint32_t* crc) { return TRI_ERROR_OUT_OF_MEMORY; } - fin = fopen(path, "rb"); + fin = TRI_FOPEN(path, "rb"); if (fin == nullptr) { TRI_Free(buffer); @@ -1901,7 +1922,7 @@ static std::string getTempPath() { // .......................................................................... #define LOCAL_MAX_PATH_BUFFER 2049 - TCHAR tempPathName[LOCAL_MAX_PATH_BUFFER]; + wchar_t tempPathName[LOCAL_MAX_PATH_BUFFER]; DWORD dwReturnValue = 0; // .......................................................................... // Attempt to locate the path where the users temporary files are stored @@ -1918,15 +1939,17 @@ static std::string getTempPath() { The path specified by the USERPROFILE environment variable. The Windows directory. */ - dwReturnValue = GetTempPath(LOCAL_MAX_PATH_BUFFER, tempPathName); + dwReturnValue = GetTempPathW(LOCAL_MAX_PATH_BUFFER, tempPathName); if ((dwReturnValue > LOCAL_MAX_PATH_BUFFER) || (dwReturnValue == 0)) { // something wrong - LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "GetTempPathA failed: LOCAL_MAX_PATH_BUFFER=" + LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "GetTempPathW failed: LOCAL_MAX_PATH_BUFFER=" << LOCAL_MAX_PATH_BUFFER << ":dwReturnValue=" << dwReturnValue; } - std::string result(tempPathName); + UnicodeString tmpPathW(tempPathName, dwReturnValue); + std::string result; + tmpPathW.toUTF8String(result); // ........................................................................... // Whether or not UNICODE is defined, we assume that the temporary file name // fits in the ascii set of characters. This is a small compromise so that @@ -1947,8 +1970,14 @@ static std::string getTempPath() { } static int mkDTemp(char* s, size_t bufferSize) { - auto rc = _mktemp_s(s, bufferSize); + std::string out; + UnicodeString sw(s); + // this will overwrite the _XXX part of the string: + auto rc = _wmktemp_s((wchar_t*)sw.getTerminatedBuffer(), bufferSize); if (rc == 0) { + // if it worked out, we need to return the utf8 version: + sw.toUTF8String(out); + memcpy(s, out.c_str(), bufferSize); rc = TRI_MKDIR(s, 0700); } return rc; @@ -2118,7 +2147,7 @@ int TRI_GetTempName(char const* directory, std::string& result, bool createFile, errorMessage = std::string("Tempfile already exists! ") + filename; } else { if (createFile) { - FILE* fd = fopen(filename.c_str(), "wb"); + FILE* fd = TRI_FOPEN(filename.c_str(), "wb"); if (fd != nullptr) { fclose(fd); @@ -2313,7 +2342,8 @@ int TRI_CreateDatafile(std::string const& filename, size_t maximalSize) { bool TRI_PathIsAbsolute(std::string const& path) { #if _WIN32 - return !PathIsRelative(path.c_str()); + UnicodeString upath(path.c_str(), (uint16_t) path.length()); + return !PathIsRelativeW(upath.getTerminatedBuffer()); #else return (!path.empty()) && path.c_str()[0] == '/'; #endif @@ -2334,3 +2364,27 @@ void TRI_InitializeFiles() { void TRI_ShutdownFiles() {} + +bool TRI_GETENV(char const* which, std::string &value) { +#ifdef _WIN32 + UnicodeString uwhich(which); + wchar_t const *v = _wgetenv(uwhich.getTerminatedBuffer()); + + if (v == nullptr) { + return false; + } + value.clear(); + UnicodeString vu(v); + vu.toUTF8String(value); + return true; +#else + char const* v = getenv(which); + + if (v == nullptr) { + return false; + } + value.clear(); + value = v; + return true; +#endif +} diff --git a/lib/Basics/files.h b/lib/Basics/files.h index 8ab81a50fe..a974dbd24d 100644 --- a/lib/Basics/files.h +++ b/lib/Basics/files.h @@ -364,4 +364,9 @@ void TRI_InitializeFiles(); void TRI_ShutdownFiles(); +//////////////////////////////////////////////////////////////////////////////// +/// @brief if which is found, value is overwriten, true returned. +//////////////////////////////////////////////////////////////////////////////// + +bool TRI_GETENV(char const* which, std::string &value); #endif diff --git a/lib/Basics/operating-system.h b/lib/Basics/operating-system.h index 2769d263fb..5df4bf75cf 100644 --- a/lib/Basics/operating-system.h +++ b/lib/Basics/operating-system.h @@ -172,6 +172,7 @@ #define TRI_LSEEK ::lseek #define TRI_MKDIR(a, b) ::mkdir((a), (b)) #define TRI_OPEN(a, b) ::open((a), (b)) +#define TRI_FOPEN(a, b) ::fopen((a), (b)) #define TRI_READ ::read #define TRI_RMDIR ::rmdir #define TRI_STAT ::stat @@ -192,6 +193,7 @@ #define TRI_LAST_ERROR_STR ::strerror(errno) #define TRI_SYSTEM_ERROR() \ {} +#define TRI_GET_ARGV(ARGC, ARGV) // sockets @@ -329,6 +331,7 @@ #define TRI_LSEEK ::lseek #define TRI_MKDIR(a, b) ::mkdir((a), (b)) #define TRI_OPEN(a, b) ::open((a), (b)) +#define TRI_FOPEN(a, b) ::fopen((a), (b)) #define TRI_READ ::read #define TRI_RMDIR ::rmdir #define TRI_STAT ::stat @@ -349,6 +352,7 @@ #define TRI_LAST_ERROR_STR ::strerror(errno) #define TRI_SYSTEM_ERROR() \ {} +#define TRI_GET_ARGV(ARGC, ARGV) // sockets @@ -473,6 +477,7 @@ #define TRI_LSEEK ::lseek #define TRI_MKDIR(a, b) ::mkdir((a), (b)) #define TRI_OPEN(a, b) ::open((a), (b)) +#define TRI_FOPEN(a, b) ::fopen((a), (b)) #define TRI_READ ::read #define TRI_RMDIR ::rmdir #define TRI_STAT ::stat @@ -493,6 +498,7 @@ #define TRI_LAST_ERROR_STR ::strerror(errno) #define TRI_SYSTEM_ERROR() \ {} +#define TRI_GET_ARGV(ARGC, ARGV) // sockets @@ -631,6 +637,7 @@ #define TRI_LSEEK ::lseek #define TRI_MKDIR(a, b) ::mkdir((a), (b)) #define TRI_OPEN(a, b) ::open((a), (b)) +#define TRI_FOPEN(a, b) ::fopen((a), (b)) #define TRI_READ ::read #define TRI_RMDIR ::rmdir #define TRI_STAT ::stat @@ -651,6 +658,7 @@ #define TRI_LAST_ERROR_STR ::strerror(errno) #define TRI_SYSTEM_ERROR() \ {} +#define TRI_GET_ARGV(ARGC, ARGV) // sockets @@ -704,7 +712,6 @@ #include #include #include - // available include files #define TRI_HAVE_DIRECT_H 1 @@ -829,18 +836,13 @@ typedef unsigned char bool; #define O_RDONLY _O_RDONLY -#define TRI_CHDIR ::_chdir #define TRI_CLOSE ::_close #define TRI_CREATE(a, b, c) TRI_createFile((a), (b), (c)) #define TRI_FSTAT ::_fstat64 -#define TRI_GETCWD ::_getcwd #define TRI_LSEEK ::_lseeki64 -#define TRI_MKDIR(a, b) ::_mkdir((a)) +#define TRI_MKDIR(a, b) TRI_MKDIR_WIN32(a) #define TRI_OPEN(a, b) TRI_OPEN_WIN32((a), (b)) #define TRI_READ ::_read -#define TRI_RMDIR ::_rmdir -#define TRI_STAT ::_stat64 -#define TRI_UNLINK ::_unlink #define TRI_WRITE ::_write #define TRI_FDOPEN(a, b) ::_fdopen((a), (b)) @@ -853,6 +855,17 @@ typedef unsigned char bool; #define TRI_ERRORBUF char windowsErrorBuf[256] = ""; #define TRI_GET_ERRORBUF windowsErrorBuf +#define TRI_GET_ARGV(ARGC, ARGV) TRI_GET_ARGV_WIN(ARGC, ARGV) + +// Implemented wrappers in win-utils.cpp: +FILE* TRI_FOPEN(char const* filename, char const* mode); +int TRI_CHDIR(char const* dirname); +int TRI_STAT(char const* path, TRI_stat_t* buffer); +char* TRI_GETCWD(char* buffer, int maxlen); +int TRI_MKDIR_WIN32(char const* dirname); +int TRI_RMDIR(char const* dirname); +int TRI_UNLINK(char const* filename); +void TRI_GET_ARGV_WIN(int& argc, char** argv); // system error string macro requires ERRORBUF to instantiate its buffer before. diff --git a/lib/Basics/process-utils.cpp b/lib/Basics/process-utils.cpp index 43dbf8ae7b..a56c0c25a0 100644 --- a/lib/Basics/process-utils.cpp +++ b/lib/Basics/process-utils.cpp @@ -44,6 +44,7 @@ #ifdef _WIN32 #include #include +#include #endif #include "Logger/Logger.h" @@ -326,20 +327,57 @@ static int appendQuotedArg(TRI_string_buffer_t* buf, char const* p) { return TRI_ERROR_NO_ERROR; } -static char* makeWindowsArgs(ExternalProcess* external) { - TRI_string_buffer_t* buf; + +static int wAppendQuotedArg(std::wstring &buf, wchar_t const* p) { + buf += L'"'; + + while (*p != 0) { + unsigned int i; + unsigned int NumberBackslashes = 0; + wchar_t const* q = p; + while (*q == L'\\') { + ++q; + ++NumberBackslashes; + } + if (*q == 0) { + // Escape all backslashes, but let the terminating + // double quotation mark we add below be interpreted + // as a metacharacter. + for (i = 0; i < NumberBackslashes; i++) { + buf += L'\\'; + buf += L'\\'; + } + break; + } else if (*q == L'"') { + // Escape all backslashes and the following + // double quotation mark. + for (i = 0; i < NumberBackslashes; i++) { + buf += L'\\'; + buf += L'\\'; + } + buf += L'\\'; + buf += *q; + } else { + // Backslashes aren't special here. + for (i = 0; i < NumberBackslashes; i++) { + buf += L'\\'; + } + buf += *q; + } + p = ++q; + } + buf += L'"'; + return TRI_ERROR_NO_ERROR; +} +static std::wstring makeWindowsArgs(ExternalProcess* external) { size_t i; int err = TRI_ERROR_NO_ERROR; - char* res; - - buf = TRI_CreateStringBuffer(); - if (buf == nullptr) { - return nullptr; - } + std::wstring res; if (( external->_executable.find('/') == std::string::npos) && ( external->_executable.find('\\') == std::string::npos)) { // oK, this is a binary without path, start the lookup. + // This will most probably break with non-ascii paths. char buf[MAX_PATH]; char *pBuf; DWORD n; @@ -349,34 +387,32 @@ static char* makeWindowsArgs(ExternalProcess* external) { } } - TRI_ReserveStringBuffer(buf, 1024); - err = appendQuotedArg(buf, external->_executable.c_str()); + UnicodeString uwargs(external->_executable.c_str()); + + err = wAppendQuotedArg(res, uwargs.getTerminatedBuffer()); if (err != TRI_ERROR_NO_ERROR) { - TRI_FreeStringBuffer(buf); return nullptr; } for (i = 1; i < external->_numberArguments; i++) { - err = TRI_AppendCharStringBuffer(buf, ' '); + res += L' '; + uwargs = external->_arguments[i]; + err = wAppendQuotedArg(res, uwargs.getTerminatedBuffer()); if (err != TRI_ERROR_NO_ERROR) { - TRI_FreeStringBuffer(buf); return nullptr; } - err = appendQuotedArg(buf, external->_arguments[i]); } - res = TRI_StealStringBuffer(buf); - TRI_FreeStringBuffer(buf); return res; } static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) { - char* args; + std::wstring args; PROCESS_INFORMATION piProcInfo; - STARTUPINFO siStartInfo; + STARTUPINFOW siStartInfo; BOOL bFuncRetn = FALSE; TRI_ERRORBUF; args = makeWindowsArgs(external); - if (args == nullptr) { + if (args.length() == 0) { LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "execute of '" << external->_executable << "' failed making args"; return false; @@ -386,8 +422,8 @@ static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) { ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); // set up members of the STARTUPINFO structure - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); + ZeroMemory(&siStartInfo, sizeof(STARTUPINFOW)); + siStartInfo.cb = sizeof(STARTUPINFOW); siStartInfo.dwFlags = STARTF_USESTDHANDLES; siStartInfo.hStdInput = rd ? rd : nullptr; @@ -395,18 +431,16 @@ static bool startProcess(ExternalProcess* external, HANDLE rd, HANDLE wr) { siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); // create the child process - bFuncRetn = CreateProcess(NULL, - args, // command line - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - CREATE_NEW_PROCESS_GROUP, // creation flags - NULL, // use parent's environment - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &piProcInfo); // receives PROCESS_INFORMATION - - TRI_Free(args); + bFuncRetn = CreateProcessW(NULL, + (LPWSTR)args.c_str(), // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + CREATE_NEW_PROCESS_GROUP, // creation flags + NULL, // use parent's environment + NULL, // use parent's current directory + &siStartInfo, // STARTUPINFO pointer + &piProcInfo); // receives PROCESS_INFORMATION if (bFuncRetn == FALSE) { TRI_SYSTEM_ERROR(); diff --git a/lib/Basics/tri-zip.cpp b/lib/Basics/tri-zip.cpp index 7c4ef2e861..79932517b8 100644 --- a/lib/Basics/tri-zip.cpp +++ b/lib/Basics/tri-zip.cpp @@ -26,6 +26,7 @@ #include "Basics/files.h" #include "Basics/FileUtils.h" #include "Basics/tri-strings.h" +#include "Basics/Common.h" #include "Zip/unzip.h" #include "Zip/zip.h" @@ -147,7 +148,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize, } // try to write the outfile - fout = fopen(fullPath.c_str(), "wb"); + fout = TRI_FOPEN(fullPath.c_str(), "wb"); // cannot write to outfile. this may be due to the target directory missing if (fout == nullptr && !skipPaths && @@ -166,7 +167,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize, *(filenameWithoutPath - 1) = c; // try again - fout = fopen(fullPath.c_str(), "wb"); + fout = TRI_FOPEN(fullPath.c_str(), "wb"); } else if (fout == nullptr) { // try to create the target directory recursively // strip filename so we only have the directory name @@ -178,7 +179,7 @@ static int ExtractCurrentFile(unzFile uf, void* buffer, size_t const bufferSize, } // try again - fout = fopen(fullPath.c_str(), "wb"); + fout = TRI_FOPEN(fullPath.c_str(), "wb"); } if (fout == nullptr) { @@ -340,7 +341,7 @@ int TRI_ZipFile(char const* filename, char const* dir, break; } - FILE* fin = fopen(fullfile.c_str(), "rb"); + FILE* fin = TRI_FOPEN(fullfile.c_str(), "rb"); if (fin == nullptr) { break; @@ -394,7 +395,7 @@ int TRI_Adler32(char const* filename, uint32_t& checksum) { } TRI_DEFER(TRI_CLOSE(fd)); - struct TRI_STAT statbuf; + TRI_stat_t statbuf; int res = TRI_FSTAT(fd, &statbuf); if (res < 0) { TRI_ERRORBUF; diff --git a/lib/Basics/win-utils.cpp b/lib/Basics/win-utils.cpp index 0ab30f6b64..9d179b5bed 100644 --- a/lib/Basics/win-utils.cpp +++ b/lib/Basics/win-utils.cpp @@ -24,6 +24,15 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "win-utils.h" @@ -33,14 +42,13 @@ #include #include #include -#include -#include #include "Logger/Logger.h" #include "Basics/files.h" #include "Basics/StringUtils.h" #include "Basics/tri-strings.h" #include "Basics/directories.h" +#include "Basics/Common.h" using namespace arangodb::basics; @@ -181,11 +189,16 @@ int initializeWindows(const TRI_win_initialize_e initializeWhat, int TRI_createFile(char const* filename, int openFlags, int modeFlags) { HANDLE fileHandle; int fileDescriptor; - + UnicodeString fn(filename); + fileHandle = - CreateFileA(filename, GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, - (openFlags & O_APPEND) ? OPEN_ALWAYS : CREATE_NEW, 0, NULL); + CreateFileW(fn.getTerminatedBuffer(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + (openFlags & O_APPEND) ? OPEN_ALWAYS : CREATE_NEW, + 0, + NULL); if (fileHandle == INVALID_HANDLE_VALUE) { return -1; @@ -228,8 +241,12 @@ int TRI_OPEN_WIN32(char const* filename, int openFlags) { break; } - fileHandle = CreateFileA( - filename, mode, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + UnicodeString fn(filename); + fileHandle = CreateFileW(fn.getTerminatedBuffer(), + mode, + FILE_SHARE_DELETE | + FILE_SHARE_READ | + FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); if (fileHandle == INVALID_HANDLE_VALUE) { @@ -241,6 +258,64 @@ int TRI_OPEN_WIN32(char const* filename, int openFlags) { return fileDescriptor; } + +FILE* TRI_FOPEN(char const* filename, char const* mode) { + UnicodeString fn(filename); + UnicodeString umod(mode); + return _wfopen(fn.getTerminatedBuffer(), umod.getTerminatedBuffer()); +} + + +int TRI_CHDIR(char const* dirname) { + UnicodeString dn(dirname); + return ::_wchdir(dn.getTerminatedBuffer()); +} + +int TRI_STAT(char const* path, TRI_stat_t* buffer) { + UnicodeString p(path); + auto rc = ::_wstat64(p.getTerminatedBuffer(), buffer); + return rc; +} + +char *TRI_GETCWD(char* buffer, int maxlen){ + char* rc = nullptr; + wchar_t* rcw; + int wBufLen = maxlen; + wchar_t* wbuf = (wchar_t*)malloc(wBufLen * sizeof(wchar_t)); + + if (wbuf == nullptr) { + return nullptr; + } + rcw = ::_wgetcwd(wbuf, wBufLen); + + if (rcw != nullptr) { + std::string rcs; + + UnicodeString d(wbuf, static_cast(wcslen(wbuf))); + d.toUTF8String(rcs); + if (rcs.length() + 1 < maxlen) { + memcpy(buffer, rcs.c_str(), rcs.length() + 1); + rc = buffer; + } + } + free(wbuf); + return rc; +} + +int TRI_MKDIR_WIN32(char const* dirname) { + UnicodeString dir(dirname); + return ::_wmkdir(dir.getTerminatedBuffer()); +} + +int TRI_RMDIR(char const* dirname) { + UnicodeString dir(dirname); + return ::_wrmdir(dir.getTerminatedBuffer()); +} +int TRI_UNLINK(char const* filename) { + UnicodeString fn(filename); + return ::_wunlink(fn.getTerminatedBuffer()); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief converts a Windows error to a *nix system error //////////////////////////////////////////////////////////////////////////////// @@ -411,7 +486,7 @@ int TRI_MapSystemError(DWORD error) { static HANDLE hEventLog = INVALID_HANDLE_VALUE; bool TRI_InitWindowsEventLog(void) { - hEventLog = RegisterEventSource(NULL, "ArangoDB"); + hEventLog = RegisterEventSourceW(NULL, L"ArangoDB"); if (NULL == hEventLog) { // well, fail then. return false; @@ -424,13 +499,6 @@ void TRI_CloseWindowsEventlog(void) { hEventLog = INVALID_HANDLE_VALUE; } -//////////////////////////////////////////////////////////////////////////////// -/// @brief logs a message to the windows event log. -/// we rather are keen on logging something at all then on being able to work -/// with fancy dynamic buffers; thus we work with a static buffer. -/// the arango internal logging will handle that usually. -//////////////////////////////////////////////////////////////////////////////// - // No clue why there is no header for these... #define MSG_INVALID_COMMAND ((DWORD)0xC0020100L) #define UI_CATEGORY ((WORD)0x00000003L) @@ -447,10 +515,27 @@ void TRI_LogWindowsEventlog(char const* func, char const* file, int line, DWORD len = _snprintf(buf, sizeof(buf) - 1, "%s", message.c_str()); buf[sizeof(buf) - 1] = '\0'; + UnicodeString ubufs[]{ + UnicodeString(buf, len), + UnicodeString(file), + UnicodeString(func), + UnicodeString(linebuf) + }; + LPCWSTR buffers[] = { + ubufs[0].getTerminatedBuffer(), + ubufs[1].getTerminatedBuffer(), + ubufs[2].getTerminatedBuffer(), + ubufs[3].getTerminatedBuffer(), + nullptr + }; // Try to get messages through to windows syslog... - if (!ReportEvent(hEventLog, EVENTLOG_ERROR_TYPE, UI_CATEGORY, - MSG_INVALID_COMMAND, NULL, 4, 0, (LPCSTR*)logBuffers, - NULL)) { + if (!ReportEventW(hEventLog, + EVENTLOG_ERROR_TYPE, + UI_CATEGORY, + MSG_INVALID_COMMAND, + NULL, + 4, 0, + buffers, NULL)) { // well, fail then... } } @@ -468,10 +553,27 @@ void TRI_LogWindowsEventlog(char const* func, char const* file, int line, DWORD len = _vsnprintf(buf, sizeof(buf) - 1, fmt, ap); buf[sizeof(buf) - 1] = '\0'; + UnicodeString ubufs[]{ + UnicodeString(buf, len), + UnicodeString(file), + UnicodeString(func), + UnicodeString(linebuf) + }; + LPCWSTR buffers[] = { + ubufs[0].getTerminatedBuffer(), + ubufs[1].getTerminatedBuffer(), + ubufs[2].getTerminatedBuffer(), + ubufs[3].getTerminatedBuffer(), + nullptr + }; // Try to get messages through to windows syslog... - if (!ReportEvent(hEventLog, EVENTLOG_ERROR_TYPE, UI_CATEGORY, - MSG_INVALID_COMMAND, NULL, 4, 0, (LPCSTR*)logBuffers, - NULL)) { + if (!ReportEventW(hEventLog, + EVENTLOG_ERROR_TYPE, + UI_CATEGORY, + MSG_INVALID_COMMAND, + NULL, + 4, 0, + buffers, NULL)) { // well, fail then... } } @@ -656,3 +758,29 @@ std::string getFileNameFromHandle(HANDLE fileHandle) { } return std::string((LPCTSTR)CString(FileInformation->FileName)); } + +static std::vector argVec; + +void TRI_GET_ARGV_WIN(int& argc, char** argv) { + auto wargStr = GetCommandLineW(); + + // if you want your argc in unicode, all you gonna do + // is ask: + auto wargv = CommandLineToArgvW(wargStr, &argc); + + argVec.reserve(argc); + + UnicodeString buf; + std::string uBuf; + for (int i = 0; i < argc; i++) { + uBuf.clear(); + // convert one UTF16 argument to utf8: + buf = wargv[i]; + buf.toUTF8String(uBuf); + // memorize the utf8 value to keep the instance: + argVec.push_back(uBuf); + + // Now overwrite our original argc entry with the utf8 one: + argv[i] = (char*) argVec[i].c_str(); + } +} diff --git a/lib/Logger/LogAppenderFile.cpp b/lib/Logger/LogAppenderFile.cpp index 7c260e9498..2dbbc9e0cf 100644 --- a/lib/Logger/LogAppenderFile.cpp +++ b/lib/Logger/LogAppenderFile.cpp @@ -202,7 +202,7 @@ void LogAppenderFile::reopenAll() { backup.append(".old"); FileUtils::remove(backup); - FileUtils::rename(filename, backup); + TRI_RenameFile(filename.c_str(), backup.c_str()); // open new log file int fd = TRI_TRACKED_CREATE_FILE(filename.c_str(), @@ -210,7 +210,7 @@ void LogAppenderFile::reopenAll() { S_IRUSR | S_IWUSR | S_IRGRP); if (fd < 0) { - FileUtils::rename(backup, filename); + TRI_RenameFile(backup.c_str(), filename.c_str()); continue; } diff --git a/lib/ProgramOptions/IniFileParser.h b/lib/ProgramOptions/IniFileParser.h index 362c133239..08ba3eea2f 100644 --- a/lib/ProgramOptions/IniFileParser.h +++ b/lib/ProgramOptions/IniFileParser.h @@ -25,10 +25,11 @@ #include "Basics/FileUtils.h" -#include #include +#include #include "ProgramOptions/ProgramOptions.h" +#include namespace arangodb { namespace options { @@ -67,25 +68,25 @@ class IniFileParser { return _options->fail( "unable to open configuration file: no configuration file specified"); } - - std::ifstream ifs(filename, std::ifstream::in); - - if (!ifs.is_open()) { - return _options->fail("unable to open configuration file '" + filename + - "'"); + std::string buf; + try { + buf = arangodb::basics::FileUtils::slurp(filename); + } + catch (arangodb::basics::Exception const& ex) { + return _options->fail(std::string("Couldn't open configuration file: '") + + filename + "' - " + ex.what()); + return true; } - bool isCommunity = false; bool isEnterprise = false; std::string currentSection; size_t lineNumber = 0; - - while (ifs.good()) { - std::string line; + + std::istringstream iss(buf); + for (std::string line; std::getline(iss, line); ) { + basics::StringUtils::trimInPlace(line); ++lineNumber; - std::getline(ifs, line); - if (std::regex_match(line, _matchers.comment)) { // skip over comments continue; diff --git a/lib/ProgramOptions/Translator.cpp b/lib/ProgramOptions/Translator.cpp index 73e9800e72..086a816e26 100644 --- a/lib/ProgramOptions/Translator.cpp +++ b/lib/ProgramOptions/Translator.cpp @@ -76,24 +76,21 @@ std::string arangodb::options::EnvironmentTranslator(std::string const& value, if (q < e) { std::string k = std::string(t, q); - char const* v = getenv(k.c_str()); std::string vv; - - if (v != nullptr && *v == '\0') { - v = nullptr; - } - - if (v == nullptr) { + if (!TRI_GETENV(k.c_str(), vv) || (vv.length() == 0)) { auto iter = environment.find(k); if (iter != environment.end()) { - v = iter->second.c_str(); + vv = iter->second; } } - if (v == nullptr) { + if (vv.length() == 0) { + if (k == "PID") { + vv = std::to_string(Thread::currentProcessId()); + } #if _WIN32 - if (TRI_EqualString(k.c_str(), "ROOTDIR")) { + else if (k == "ROOTDIR") { vv = TRI_LocateInstallDirectory(nullptr, binaryPath); if (!vv.empty()) { @@ -103,13 +100,8 @@ std::string arangodb::options::EnvironmentTranslator(std::string const& value, vv.pop_back(); } } - } else + } #endif - if (TRI_EqualString(k.c_str(), "PID")) { - vv = std::to_string(Thread::currentProcessId()); - } - } else { - vv = v; } result += vv; diff --git a/lib/V8/v8-utils.cpp b/lib/V8/v8-utils.cpp index 9fe248f8d5..432d2f2e30 100644 --- a/lib/V8/v8-utils.cpp +++ b/lib/V8/v8-utils.cpp @@ -2695,10 +2695,16 @@ static void JS_Write(v8::FunctionCallbackInfo const& args) { if (args.Length() < 2) { TRI_V8_THROW_EXCEPTION_USAGE("write(, )"); } - - TRI_Utf8ValueNFC name(args[0]); - - if (*name == nullptr) { +#if _WIN32 // the wintendo needs utf16 filenames + v8::String::Value str(args[0]); + std::wstring name { + reinterpret_cast(*str), + static_cast(str.length())}; +#else + TRI_Utf8ValueNFC str(args[0]); + std::string name(*str, str.length()); +#endif + if (name.length() == 0) { TRI_V8_THROW_TYPE_ERROR(" must be a string"); } @@ -2722,7 +2728,7 @@ static void JS_Write(v8::FunctionCallbackInfo const& args) { errno = 0; // disable exceptions in the stream object: file.exceptions(std::ifstream::goodbit); - file.open(*name, std::ios::out | std::ios::binary); + file.open(name, std::ios::out | std::ios::binary); if (file.is_open() && file.good()) { file.write(data, size); @@ -2752,7 +2758,7 @@ static void JS_Write(v8::FunctionCallbackInfo const& args) { errno = 0; // disable exceptions in the stream object: file.exceptions(std::ifstream::goodbit); - file.open(*name, std::ios::out | std::ios::binary); + file.open(name, std::ios::out | std::ios::binary); if (file.is_open() && file.good()) { file << *content; diff --git a/lib/Zip/iowin32.cpp b/lib/Zip/iowin32.cpp index 5d46ed26fc..0cf73eb414 100644 --- a/lib/Zip/iowin32.cpp +++ b/lib/Zip/iowin32.cpp @@ -12,7 +12,7 @@ For more info read MiniZip_info.txt */ - +#include #include #include "zlib.h" @@ -108,10 +108,16 @@ win32_open64_file_funcA(voidpf opaque, const void* filename, int mode) { win32_translate_open_mode(mode, &dwDesiredAccess, &dwCreationDisposition, &dwShareMode, &dwFlagsAndAttributes); - if ((filename != NULL) && (dwDesiredAccess != 0)) - hFile = CreateFileA((LPCSTR)filename, dwDesiredAccess, dwShareMode, NULL, - dwCreationDisposition, dwFlagsAndAttributes, NULL); - + if ((filename != NULL) && (dwDesiredAccess != 0)) { + UnicodeString fn((LPSTR)filename); + hFile = CreateFileW(fn.getTerminatedBuffer(), + dwDesiredAccess, + dwShareMode, + NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + } return win32_build_iowin(hFile); } diff --git a/tests/main.cpp b/tests/main.cpp index 90ed5a1eb4..aa124a9bc2 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -50,6 +50,7 @@ private: char const* ARGV0 = ""; int main(int argc, char* argv[]) { + TRI_GET_ARGV(argc, argv); int subargc = 0; char **subargv = (char**)malloc(sizeof(char*) * argc); bool logLineNumbers = false;