From 3baf63d13ee11c47d74a2a68beb75af9360161e6 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 25 Aug 2016 16:43:27 +0200 Subject: [PATCH 01/24] Fix package install paths --- cmake/packages/client/deb.txt | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/cmake/packages/client/deb.txt b/cmake/packages/client/deb.txt index dcee232749..2b232a88fe 100644 --- a/cmake/packages/client/deb.txt +++ b/cmake/packages/client/deb.txt @@ -8,9 +8,23 @@ cmake_minimum_required(VERSION 2.8) # variables from the main build have to be explicitely forwarded: ################################################################################ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "@PROJECT_BINARY_DIR@/bin/") -set(CMAKE_INSTALL_BINDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_X ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + +set(CMAKE_INSTALL_BINDIR @CMAKE_INSTALL_BINDIR@) +set(CMAKE_INSTALL_FULL_BINDIR @CMAKE_INSTALL_FULL_BINDIR@) + +set(CMAKE_INSTALL_DATAROOTDIR @CMAKE_INSTALL_DATAROOTDIR@) +set(CMAKE_INSTALL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_DATAROOTDIR_ARANGO@) +set(CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO @CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO@) + +set(CMAKE_INSTALL_DIR @CMAKE_INSTALL_DIR@) +set(CMAKE_INSTALL_PREFIX @CMAKE_INSTALL_PREFIX@) + +set(CMAKE_INSTALL_SYSCONFDIR @CMAKE_INSTALL_SYSCONFDIR@) +set(CMAKE_INSTALL_SYSCONFDIR_ARANGO @CMAKE_INSTALL_SYSCONFDIR_ARANGO@) +set(CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO @CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO@) + ################################################################################ # Substitute the install binaries: ################################################################################ @@ -29,21 +43,20 @@ set(ARANGODB_SOURCE_DIR @ARANGODB_SOURCE_DIR@) set(ARANGODB_VERSION @ARANGODB_VERSION@) set(ARANGODB_PACKAGE_CONTACT @ARANGODB_PACKAGE_CONTACT@) set(ARANGODB_PACKAGE_REVISION @ARANGODB_PACKAGE_REVISION@) - -set(CMAKE_INSTALL_FULL_BINDIR @CMAKE_INSTALL_FULL_BINDIR@) +set(ARANGODB_PACKAGE_VENDOR @ARANGODB_PACKAGE_VENDOR@) set(CMAKE_TARGET_ARCHITECTURES @CMAKE_TARGET_ARCHITECTURES@) -set(CMAKE_INSTALL_SYSCONFDIR_ARANGO @CMAKE_INSTALL_SYSCONFDIR_ARANGO@) -set(CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO @CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO@) - set(ORIGINAL_SOURCE_DIR @PROJECT_SOURCE_DIR@) set(PROJECT_SOURCE_DIR @PROJECT_SOURCE_DIR@) + ################################################################################ # Get the final values for cpack: ################################################################################ set(CPACK_PACKAGE_VERSION "${ARANGODB_VERSION}") set(CPACK_PACKAGE_NAME "arangodb3-client") +set(CPACK_DEBIAN_PACKAGE_SECTION "shell") +set(CPACK_PACKAGE_VENDOR ${ARANGODB_PACKAGE_VENDOR}) set(CPACK_PACKAGE_CONTACT ${ARANGODB_PACKAGE_CONTACT}) set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE") set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${ARANGODB_URL_INFO_ABOUT}) @@ -51,6 +64,10 @@ set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) set(CPACK_DEBIAN_PACKAGE_CONFLICTS "arangodb, arangodb3") set(CPACK_DEBIAN_COMPRESSION_TYPE "xz") set(CPACK_COMPONENTS_ALL debian-extras) +set(CPACK_GENERATOR "DEB") +set(CPACK_SET_DESTDIR ON) + +set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) file(READ "${PROJECT_SOURCE_DIR}/Installation/debian/client_packagedesc.txt" CPACK_DEBIAN_PACKAGE_DESCRIPTION) @@ -73,12 +90,7 @@ set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${AR ################################################################################ # Install the external files into the package directory: ################################################################################ -include(${PROJECT_SOURCE_DIR}/cmake/GNUInstallDirs.cmake) -set(CMAKE_INSTALL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_SYSCONFDIR}/arangodb3" CACHE PATH "read-only single-machine data (etc)") -set(CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_FULL_SYSCONFDIR}/arangodb3" CACHE PATH "read-only single-machine data (etc)") -set(CMAKE_INSTALL_DATAROOTDIR_ARANGO "${CMAKE_INSTALL_DATAROOTDIR}/arangodb3" CACHE PATH "read-only data (share)") -set(CMAKE_INSTALL_FULL_DATAROOTDIR_ARANGO "${CMAKE_INSTALL_FULL_DATAROOTDIR}/arangodb3" CACHE PATH "read-only data (share)") set(INSTALL_MACROS_NO_TARGET_INSTALL TRUE) include(${ORIGINAL_SOURCE_DIR}/cmake/InstallMacros.cmake) From efc949426f1e4d76aac599bd4a3ace3e6171193f Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 25 Aug 2016 17:23:54 +0200 Subject: [PATCH 02/24] Fix install permissions of binaries. --- arangosh/install.cmake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arangosh/install.cmake b/arangosh/install.cmake index 5ef63158e4..c3a77f1684 100644 --- a/arangosh/install.cmake +++ b/arangosh/install.cmake @@ -3,32 +3,32 @@ # we can't use RUNTIME DESTINATION here. install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOBENCH}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_config(arangobench) install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGODUMP}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_config(arangodump) install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOIMP}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_config(arangoimp) install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGORESTORE}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_config(arangorestore) install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOSH}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_config(arangosh) install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOVPACK}${CMAKE_EXECUTABLE_SUFFIX} + PROGRAMS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY_X}/${BIN_ARANGOVPACK}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${CMAKE_INSTALL_BINDIR}) install_command_alias(${BIN_ARANGOSH} From f39ec1c5919cdea5ea385a4e595cfb33e63b9436 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 25 Aug 2016 17:35:29 +0200 Subject: [PATCH 03/24] Windows install: no backslash strings in cmake. --- cmake/ArangoDBInstall.cmake | 2 -- lib/Basics/directories.h.in | 2 +- lib/Basics/win-utils.cpp | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/ArangoDBInstall.cmake b/cmake/ArangoDBInstall.cmake index 0dbb5b4ee5..c333e701cd 100644 --- a/cmake/ArangoDBInstall.cmake +++ b/cmake/ArangoDBInstall.cmake @@ -16,8 +16,6 @@ endif () set(CMAKE_INSTALL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_SYSCONFDIR}/arangodb3") set(CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO "${CMAKE_INSTALL_FULL_SYSCONFDIR}/arangodb3") -file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO}" ETCDIR_NATIVE) - # database directory FILE(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/var/lib/arangodb3") diff --git a/lib/Basics/directories.h.in b/lib/Basics/directories.h.in index 5408504c14..50f13e3013 100644 --- a/lib/Basics/directories.h.in +++ b/lib/Basics/directories.h.in @@ -1,5 +1,5 @@ #define LOCCAL_STATE_DIR "@CMAKE_INSTALL_FULL_LOCALSTATEDIR@" -#define _SYSCONFDIR_ "@ETCDIR_NATIVE@" +#define _SYSCONFDIR_ "@CMAKE_INSTALL_FULL_SYSCONFDIR_ARANGO@" #define STARTUP_DIRECTORY "@PKGDATADIR@/js" #define DESTINATION_DIR "@CMAKE_INSTALL_DATAROOTDIR_ARANGO@/js" #define ICU_DESTINATION_DIRECTORY "@ICU_DT_DEST@" diff --git a/lib/Basics/win-utils.cpp b/lib/Basics/win-utils.cpp index 768cc26947..5e0bab7b40 100644 --- a/lib/Basics/win-utils.cpp +++ b/lib/Basics/win-utils.cpp @@ -325,7 +325,9 @@ void TRI_FixIcuDataEnv() { putenv(e.c_str()); } else { #ifdef _SYSCONFDIR_ - std::string e = "ICU_DATA=" + std::string(_SYSCONFDIR_) + "..\\..\\bin"; + std::string SCDIR(_SYSCONFDIR_) + SCDIR = StringUtils::replace(SCDIR, "/", "\\\\"); + std::string e = "ICU_DATA=" + SCDIR + "..\\..\\bin"; e = StringUtils::replace(e, "\\", "\\\\"); putenv(e.c_str()); #else From d37c9a751f395eb18ef5a77f80836e6636912df3 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Thu, 25 Aug 2016 17:50:08 +0200 Subject: [PATCH 04/24] fix syntax :-( --- lib/Basics/win-utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Basics/win-utils.cpp b/lib/Basics/win-utils.cpp index 5e0bab7b40..40ba03dd09 100644 --- a/lib/Basics/win-utils.cpp +++ b/lib/Basics/win-utils.cpp @@ -325,7 +325,7 @@ void TRI_FixIcuDataEnv() { putenv(e.c_str()); } else { #ifdef _SYSCONFDIR_ - std::string SCDIR(_SYSCONFDIR_) + std::string SCDIR(_SYSCONFDIR_); SCDIR = StringUtils::replace(SCDIR, "/", "\\\\"); std::string e = "ICU_DATA=" + SCDIR + "..\\..\\bin"; e = StringUtils::replace(e, "\\", "\\\\"); From 3d6810ecfe02a9043210e8a52dbd1e230566f3ed Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 26 Aug 2016 12:50:48 +0200 Subject: [PATCH 05/24] If we have relative paths that are working, make them absolute so they still work after CWD's of arangod --- lib/Basics/ArangoGlobalContext.cpp | 6 ++++++ lib/Basics/FileUtils.cpp | 10 ++++++++++ lib/Basics/FileUtils.h | 8 ++++++++ lib/Basics/files.cpp | 13 +++++++++++++ lib/Basics/files.h | 6 ++++++ 5 files changed, 43 insertions(+) diff --git a/lib/Basics/ArangoGlobalContext.cpp b/lib/Basics/ArangoGlobalContext.cpp index 18ce5ab419..cbf8ed404a 100644 --- a/lib/Basics/ArangoGlobalContext.cpp +++ b/lib/Basics/ArangoGlobalContext.cpp @@ -284,6 +284,12 @@ void ArangoGlobalContext::getCheckPath(std::string &path, const char *whichPath, LOG(ERR) << "failed to locate " << whichPath << " directory, its neither available in '" << path << "' nor in '" << directory << "'"; FATAL_ERROR_EXIT(); } + arangodb::basics::FileUtils::normalizePath(directory); path = directory; } + else { + if (!TRI_PathIsAbsolute(path)) { + arangodb::basics::FileUtils::makePathAbsolute(path); + } + } } diff --git a/lib/Basics/FileUtils.cpp b/lib/Basics/FileUtils.cpp index 21dc96ad5d..338429d84d 100644 --- a/lib/Basics/FileUtils.cpp +++ b/lib/Basics/FileUtils.cpp @@ -616,6 +616,16 @@ std::string dirname(std::string const& name) { return base; } + +void makePathAbsolute(std::string &path) { + int err = 0; + + std::string cwd = FileUtils::currentDirectory(&err); + char * p = TRI_GetAbsolutePath(path.c_str(), cwd.c_str()); + path = p; + TRI_FreeString(TRI_CORE_MEM_ZONE, p); +} + } } } diff --git a/lib/Basics/FileUtils.h b/lib/Basics/FileUtils.h index aa0a728994..97ae1bdefd 100644 --- a/lib/Basics/FileUtils.h +++ b/lib/Basics/FileUtils.h @@ -53,6 +53,14 @@ std::string removeTrailingSeparator(std::string const& name); void normalizePath(std::string& name); +//////////////////////////////////////////////////////////////////////////////// +/// @brief makes a path absolute +/// +/// path will be modified in-place +//////////////////////////////////////////////////////////////////////////////// + +void makePathAbsolute(std::string &path); + //////////////////////////////////////////////////////////////////////////////// /// @brief creates a filename //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index c0db9f4cae..cb28a932f9 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -2424,3 +2424,16 @@ void TRI_InitializeFiles() { //////////////////////////////////////////////////////////////////////////////// void TRI_ShutdownFiles() {} + + + +#if _WIN32 +bool TRI_PathIsAbsolute(const std::string &path) { + return !PathIsRelative(path.c_str()); +} + +#else +bool TRI_PathIsAbsolute(const std::string &path) { + return path.c_str()[0] == '/'; +} +#endif diff --git a/lib/Basics/files.h b/lib/Basics/files.h index 8204e3038c..4f3d900e20 100644 --- a/lib/Basics/files.h +++ b/lib/Basics/files.h @@ -369,4 +369,10 @@ void TRI_InitializeFiles(); void TRI_ShutdownFiles(); +//////////////////////////////////////////////////////////////////////////////// +/// @brief checks whether path is full qualified or relative +//////////////////////////////////////////////////////////////////////////////// + +bool TRI_PathIsAbsolute(const std::string &path); + #endif From 9afbc817a3bc0d54568d9579bca2d8a3380570d6 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 26 Aug 2016 14:06:59 +0200 Subject: [PATCH 06/24] Fix syntax error in windows client installer. --- Installation/Windows/client/Templates/NSIS.template.in | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Installation/Windows/client/Templates/NSIS.template.in b/Installation/Windows/client/Templates/NSIS.template.in index 225bd98f05..8e93494170 100755 --- a/Installation/Windows/client/Templates/NSIS.template.in +++ b/Installation/Windows/client/Templates/NSIS.template.in @@ -68,10 +68,6 @@ Function disableBackButton EnableWindow $0 0 FunctionEnd -SetShellVarContext all -FunctionEnd - - !include Sections.nsh ;--- Component support macros: --- From 1de83c020d6968ecfbeb604406bfd5688881aa7a Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 26 Aug 2016 14:17:28 +0200 Subject: [PATCH 07/24] fix windows compile problems. --- lib/Basics/files.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index cb28a932f9..5104bc5b79 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -25,6 +25,7 @@ #ifdef _WIN32 #include +#include #endif #include "Basics/directories.h" @@ -1700,13 +1701,13 @@ std::string TRI_LocateBinaryPath(char const* argv0) { std::string TRI_GetInstallRoot(std::string const& binaryPath, char const *installBinaryPath) { // First lets remove trailing (back) slashes from the bill: - long ibpLength = strlen(installBinaryPath); + size_t ibpLength = strlen(installBinaryPath); if (installBinaryPath[ibpLength - 1] == TRI_DIR_SEPARATOR_CHAR) { ibpLength --; } - long bpLength = binaryPath.length(); + size_t bpLength = binaryPath.length(); const char *pbPath = binaryPath.c_str(); if (pbPath[bpLength - 1] == TRI_DIR_SEPARATOR_CHAR) { From 63dd09ef76135f57afcc28dccbf73f4b2a827cd5 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Fri, 26 Aug 2016 14:23:17 +0200 Subject: [PATCH 08/24] Add missing windows library. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed18c9fa4a..16f99207ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,7 +321,7 @@ if (MSVC) #http://lists.boost.org/boost-users/2016/04/85968.php add_definitions("-D_ENABLE_ATOMIC_ALIGNMENT_FIX") - set(MSVC_LIBS crypt32.lib;WINMM.LIB;Ws2_32.lib) + set(MSVC_LIBS Shlwapi.lib;crypt32.lib;WINMM.LIB;Ws2_32.lib) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:CONSOLE /SAFESEH:NO /MACHINE:x64 /ignore:4099 ${BASE_LD_FLAGS}" From 4f994f70b562213068d5fc930b1eb4a29dffe721 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Fri, 26 Aug 2016 15:31:57 +0200 Subject: [PATCH 09/24] fix compile warning --- lib/Basics/files.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index 5104bc5b79..5ca44696f4 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -1708,18 +1708,18 @@ std::string TRI_GetInstallRoot(std::string const& binaryPath, } size_t bpLength = binaryPath.length(); - const char *pbPath = binaryPath.c_str(); + char const* pbPath = binaryPath.c_str(); if (pbPath[bpLength - 1] == TRI_DIR_SEPARATOR_CHAR) { - bpLength --; + --bpLength; } if (ibpLength > bpLength) { return TRI_DIR_SEPARATOR_STR; } - for (int i = 1; i < ibpLength; i ++) { - if (pbPath[bpLength -i] != installBinaryPath[ibpLength - i]) { + for (size_t i = 1; i < ibpLength; ++i) { + if (pbPath[bpLength - i] != installBinaryPath[ibpLength - i]) { return TRI_DIR_SEPARATOR_STR; } } From e07543c2e203aa27e153cad0b73f071230572141 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Fri, 26 Aug 2016 16:25:03 +0200 Subject: [PATCH 10/24] Adding Foxx access to agency --- arangod/Agency/v8-agency.cpp | 158 +++++++++++++++++++++---- arangod/RestServer/DatabaseFeature.cpp | 2 + lib/V8/v8-globals.cpp | 2 +- lib/V8/v8-globals.h | 12 +- 4 files changed, 145 insertions(+), 29 deletions(-) diff --git a/arangod/Agency/v8-agency.cpp b/arangod/Agency/v8-agency.cpp index 8f6437d9ec..5cf6be037d 100644 --- a/arangod/Agency/v8-agency.cpp +++ b/arangod/Agency/v8-agency.cpp @@ -41,16 +41,99 @@ using namespace arangodb::application_features; using namespace arangodb::basics; using namespace arangodb::consensus; -static void JS_LeadingVulpes(v8::FunctionCallbackInfo const& args) { +static void JS_EnabledAgent(v8::FunctionCallbackInfo const& args) { - v8::Isolate* isolate = args.GetIsolate(); + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + try { + ApplicationServer::getEnabledFeature("Agency"); + TRI_V8_RETURN_TRUE(); + } catch (std::exception const& e) { + TRI_V8_RETURN_FALSE(); + } + + TRI_V8_TRY_CATCH_END + +} + +static void JS_LeadingAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + Agent* agent = nullptr; try { AgencyFeature* feature = - ApplicationServer::getEnabledFeature("AgencyFeature"); + ApplicationServer::getEnabledFeature("Agency"); agent = feature->agent(); + + } catch (std::exception const& e) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, + std::string("couldn't access agency feature: ") + e.what()); + } + + v8::Handle r = v8::Object::New(isolate); + + r->Set(TRI_V8_ASCII_STRING("leading"), + v8::Boolean::New(isolate, agent->leading())); + + TRI_V8_RETURN(r); + TRI_V8_TRY_CATCH_END + +} + +static void JS_ReadAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + Agent* agent = nullptr; + try { + AgencyFeature* feature = + ApplicationServer::getEnabledFeature("Agency"); + agent = feature->agent(); + + } catch (std::exception const& e) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, + std::string("couldn't access agency feature: ") + e.what()); + } + + query_t query = std::make_shared(); + int res = TRI_V8ToVPack(isolate, *query, args[0], false); + + if (res != TRI_ERROR_NO_ERROR) { + TRI_V8_THROW_EXCEPTION(res); + } + + read_ret_t ret = agent->read(query); + + if (ret.accepted) { // Leading + TRI_V8_RETURN(TRI_VPackToV8(isolate, ret.result->slice())); + } else { // Not leading + TRI_V8_RETURN_FALSE(); + } + + TRI_V8_TRY_CATCH_END + +} + + +static void JS_WriteAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + Agent* agent = nullptr; + try { + AgencyFeature* feature = + ApplicationServer::getEnabledFeature("Agency"); + agent = feature->agent(); + } catch (std::exception const& e) { TRI_V8_THROW_EXCEPTION_MESSAGE( TRI_ERROR_INTERNAL, @@ -59,21 +142,50 @@ static void JS_LeadingVulpes(v8::FunctionCallbackInfo const& args) { v8::Handle r = v8::Object::New(isolate); + query_t query = std::make_shared(); + int res = TRI_V8ToVPack(isolate, *query, args[0], false); - r->Set(TRI_V8_ASCII_STRING("leading"), - v8::Boolean::New(isolate, agent->leading())); + if (res != TRI_ERROR_NO_ERROR) { + TRI_V8_THROW_EXCEPTION(res); + } - TRI_V8_RETURN(r); + write_ret_t ret = agent->write(query); + if (ret.accepted) { // Leading + + size_t errors = 0; + Builder body; + body.openObject(); + body.add("results", VPackValue(VPackValueType::Array)); + for (auto const& index : ret.indices) { + body.add(VPackValue(index)); + if (index == 0) { + errors++; + } + } + body.close(); body.close(); + + // Wait for commit of highest except if it is 0? + arangodb::consensus::index_t max_index = 0; + try { + max_index = + *std::max_element(ret.indices.begin(), ret.indices.end()); + } catch (std::exception const& e) { + LOG_TOPIC(WARN, Logger::AGENCY) + << e.what() << " " << __FILE__ << __LINE__; + } + + if (max_index > 0) { + agent->waitFor(max_index); + } + + TRI_V8_RETURN(TRI_VPackToV8(isolate, body.slice())); + } else { // Not leading + TRI_V8_RETURN_FALSE(); + } -} - -static void JS_ReadVulpes(v8::FunctionCallbackInfo const& args) { - -} - -static void JS_WriteVulpes(v8::FunctionCallbackInfo const& args) { - + TRI_V8_TRY_CATCH_END + } @@ -89,30 +201,32 @@ void TRI_InitV8Agency(v8::Isolate* isolate, v8::Handle context) { // ........................................................................... ft = v8::FunctionTemplate::New(isolate); - ft->SetClassName(TRI_V8_ASCII_STRING("ArangoVulpes")); + ft->SetClassName(TRI_V8_ASCII_STRING("ArangoAgent")); rt = ft->InstanceTemplate(); rt->SetInternalFieldCount(2); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("leading"), JS_LeadingVulpes); + isolate, rt, TRI_V8_ASCII_STRING("enabled"), JS_EnabledAgent); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("read"), JS_ReadVulpes); + isolate, rt, TRI_V8_ASCII_STRING("leading"), JS_LeadingAgent); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("write"), JS_WriteVulpes); + isolate, rt, TRI_V8_ASCII_STRING("read"), JS_ReadAgent); + TRI_AddMethodVocbase( + isolate, rt, TRI_V8_ASCII_STRING("write"), JS_WriteAgent); - v8g->VulpesTempl.Reset(isolate, rt); - ft->SetClassName(TRI_V8_ASCII_STRING("ArangoVuplesCtor")); + v8g->AgentTempl.Reset(isolate, rt); + ft->SetClassName(TRI_V8_ASCII_STRING("ArangoAgentCtor")); TRI_AddGlobalFunctionVocbase( - isolate, context, TRI_V8_ASCII_STRING("ArangoVuplesCtor"), + isolate, context, TRI_V8_ASCII_STRING("ArangoAgentCtor"), ft->GetFunction(), true); // register the global object v8::Handle aa = rt->NewInstance(); if (!aa.IsEmpty()) { TRI_AddGlobalVariableVocbase( - isolate, context, TRI_V8_ASCII_STRING("ArangoVuples"), aa); + isolate, context, TRI_V8_ASCII_STRING("ArangoAgent"), aa); } } diff --git a/arangod/RestServer/DatabaseFeature.cpp b/arangod/RestServer/DatabaseFeature.cpp index 4a722e038f..ff55356c32 100644 --- a/arangod/RestServer/DatabaseFeature.cpp +++ b/arangod/RestServer/DatabaseFeature.cpp @@ -24,6 +24,7 @@ #include "Basics/StringUtils.h" #include "Basics/ArangoGlobalContext.h" +#include "Agency/v8-agency.h" #include "Cluster/ServerState.h" #include "Cluster/v8-cluster.h" #include "GeneralServer/GeneralServerFeature.h" @@ -219,6 +220,7 @@ void DatabaseFeature::updateContexts() { i); TRI_InitV8Queries(isolate, context); TRI_InitV8Cluster(isolate, context); + TRI_InitV8Agency(isolate, context); }, vocbase); } diff --git a/lib/V8/v8-globals.cpp b/lib/V8/v8-globals.cpp index 46bee5e46d..63a4f0ea6b 100644 --- a/lib/V8/v8-globals.cpp +++ b/lib/V8/v8-globals.cpp @@ -28,6 +28,7 @@ TRI_v8_global_s::TRI_v8_global_s(v8::Isolate* isolate) JSVPack(), AgencyTempl(), + AgentTempl(), ClusterInfoTempl(), ServerStateTempl(), ClusterCommTempl(), @@ -35,7 +36,6 @@ TRI_v8_global_s::TRI_v8_global_s(v8::Isolate* isolate) VPackTempl(), VocbaseColTempl(), VocbaseTempl(), - VulpesTempl(), BufferTempl(), diff --git a/lib/V8/v8-globals.h b/lib/V8/v8-globals.h index 77ea333a99..f850f3c465 100644 --- a/lib/V8/v8-globals.h +++ b/lib/V8/v8-globals.h @@ -475,6 +475,12 @@ typedef struct TRI_v8_global_s { v8::Persistent AgencyTempl; + ////////////////////////////////////////////////////////////////////////////// + /// @brief local agent template + ////////////////////////////////////////////////////////////////////////////// + + v8::Persistent AgentTempl; + ////////////////////////////////////////////////////////////////////////////// /// @brief clusterinfo template ////////////////////////////////////////////////////////////////////////////// @@ -517,12 +523,6 @@ typedef struct TRI_v8_global_s { v8::Persistent VocbaseTempl; - ////////////////////////////////////////////////////////////////////////////// - /// @brief vulpes template - ////////////////////////////////////////////////////////////////////////////// - - v8::Persistent VulpesTempl; - ////////////////////////////////////////////////////////////////////////////// /// @brief TRI_vocbase_t template ////////////////////////////////////////////////////////////////////////////// From fb6e651525ad11e915ba2f7ff9019fa052f8b4f4 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Fri, 26 Aug 2016 16:28:42 +0200 Subject: [PATCH 11/24] Adding Foxx access to agency --- arangod/Agency/v8-agency.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Agency/v8-agency.cpp b/arangod/Agency/v8-agency.cpp index 5cf6be037d..3ede27d769 100644 --- a/arangod/Agency/v8-agency.cpp +++ b/arangod/Agency/v8-agency.cpp @@ -140,8 +140,6 @@ static void JS_WriteAgent(v8::FunctionCallbackInfo const& args) { std::string("couldn't access agency feature: ") + e.what()); } - v8::Handle r = v8::Object::New(isolate); - query_t query = std::make_shared(); int res = TRI_V8ToVPack(isolate, *query, args[0], false); From 1a55b449e4aa93de2cbf9c17c69c8fb128adf1f6 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Sun, 28 Aug 2016 15:36:48 +0200 Subject: [PATCH 12/24] add SYSTEM flag to boost includes to avoid warnings --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dca7baf41d..164b455102 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -708,7 +708,7 @@ if (NOT USE_BOOST_UNITTESTS) message(STATUS "BOOST unit-tests are disabled") endif () -include_directories(${Boost_INCLUDE_DIR}) +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) add_definitions(-DARANGODB_BOOST_VERSION=\"${Boost_VERSION}\") ################################################################################ From 9157debf961f35c66bbc701cf4e90ec37d0fa9da Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Sun, 28 Aug 2016 15:37:22 +0200 Subject: [PATCH 13/24] clean up perf script --- scripts/setupPerfEvents.sh | 106 ++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 44 deletions(-) diff --git a/scripts/setupPerfEvents.sh b/scripts/setupPerfEvents.sh index fea79be017..8a04f6c2ba 100755 --- a/scripts/setupPerfEvents.sh +++ b/scripts/setupPerfEvents.sh @@ -6,60 +6,78 @@ # # This script sets up performance monitoring events to measure single # document operations. Run this script with sudo when the ArangoDB -# process is already running. Then do +# process is already running: +# +# ./setupPerfEvents.sh +# +# Now you are able to recrod the event with: +# # sudo perf record -e "probe_arangod:*" -aR sleep 60 -# (to sample for 60 seconds). A file "perf.data" is written to the -# current directory. -# Dump the events in this file with +# +# The above command will get sample data for 60 seconds. A file "perf.data" is +# written to the current directory. Dump the events in this file with: +# # sudo perf script > perf.history +# # This logs the times when individual threads hit the events. # Use the program perfanalyis.cpp in this directory in the following way: +# # sudo ./perfanalyis < perf.history > perf.statistics -# This will group enter and exit events of functions together, compute -# the time spent and sort by function. -# Remove all events with +# +# This will group enter and exit events of functions together, compute the time +# spent and sort by function. When finised remove all events with: +# # sudo perf probe -d "probe_arangod:*" -# List events with +# +# List events with: +# # sudo perf probe -l +# +# -ARANGOD_EXECUTABLE=build/bin/arangod -perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" +main(){ + local ARANGOD_EXECUTABLE=build/bin/arangod -echo Adding events, this takes a few seconds... + #delete all existing events + perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" || exit 1 + + echo "Adding events, this takes a few seconds..." + + echo "Single document operations..." + addEvent insertLocal + addEvent removeLocal + addEvent modifyLocal + addEvent documentLocal + + echo "Single document operations on coordinator..." + addEvent insertCoordinator + addEvent removeCoordinator + addEvent updateCoordinator + addEvent replaceCoordinator + addEvent documentCoordinator + + echo "work method in HttpServerJob" + addEvent workHttpServerJob work@HttpServerJob.cpp + + echo "work method in RestDocumentHandler" + addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp + addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp + addEvent handleRequest handleRequest@HttpServer.cpp + addEvent handleWrite handleWrite@SocketTask.cpp + + addEvent tcp_sendmsg + addEvent tcp_recvmsg + + echo Done. +} addEvent() { - x=$1 - y=$2 - if [ "x$y" == "x" ] ; then - y=$x - fi - echo $x - perf probe -x $ARANGOD_EXECUTABLE -a $x=$y 2> /dev/null - perf probe -x $ARANGOD_EXECUTABLE -a ${x}Ret=$y%return 2> /dev/null + local name="$1" + local func="${2-"${name}"}" + + echo "setting up $name for function: $func" + perf probe -x $ARANGOD_EXECUTABLE -a $name=$func 2> /dev/null #enter function + perf probe -x $ARANGOD_EXECUTABLE -a ${name}Ret=$func%return 2> /dev/null #return form function } -echo Single document operations... -addEvent insertLocal -addEvent removeLocal -addEvent modifyLocal -addEvent documentLocal -echo Single document operations on coordinator... -addEvent insertCoordinator -addEvent removeCoordinator -addEvent updateCoordinator -addEvent replaceCoordinator -addEvent documentCoordinator - -echo work method in HttpServerJob -addEvent workHttpServerJob work@HttpServerJob.cpp - -echo work method in RestDocumentHandler -addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp -addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp -addEvent handleRequest handleRequest@HttpServer.cpp -addEvent handleWrite handleWrite@SocketTask.cpp - -addEvent tcp_sendmsg -addEvent tcp_recvmsg - -echo Done. +main "$@" From fe335baa6190b1eae49ee21ed7a6b03d7a6223f3 Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Sun, 28 Aug 2016 18:09:03 +0200 Subject: [PATCH 14/24] fix perfsetupscript --- scripts/setupPerfEvents.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/setupPerfEvents.sh b/scripts/setupPerfEvents.sh index 8a04f6c2ba..36af7a012f 100755 --- a/scripts/setupPerfEvents.sh +++ b/scripts/setupPerfEvents.sh @@ -36,10 +36,10 @@ # main(){ - local ARANGOD_EXECUTABLE=build/bin/arangod + local ARANGOD_EXECUTABLE=${1-build/bin/arangod} #delete all existing events - perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" || exit 1 + perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" echo "Adding events, this takes a few seconds..." From e1a4edeb0a89363dc4d983e1f3e8ff07ce42002c Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Sun, 28 Aug 2016 23:13:18 +0200 Subject: [PATCH 15/24] improve the perf script --- scripts/perfanalysis.cpp | 112 +++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/scripts/perfanalysis.cpp b/scripts/perfanalysis.cpp index 631fd5372f..01a635b561 100644 --- a/scripts/perfanalysis.cpp +++ b/scripts/perfanalysis.cpp @@ -1,17 +1,20 @@ // Compile with // g++ perfanalysis.cpp -o perfanalyis -std=c++11 -Wall -O3 -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include using namespace std; struct Event { + static const regex re; string threadName; int tid; string cpu; @@ -22,45 +25,27 @@ struct Event { bool isRet; Event(string& line) : isRet(false) { - char* s = strdup(line.c_str()); - char* p = strtok(s, " "); - char* q; - if (p != nullptr) { - threadName = p; - p = strtok(nullptr, " "); - tid = strtol(p, nullptr, 10); - p = strtok(nullptr, " "); - cpu = p; - p = strtok(nullptr, " "); - startTime = strtod(p, nullptr); - p = strtok(nullptr, " "); - q = strtok(nullptr, " "); - if (strcmp(q, "cs:") == 0) { - free(s); - return; - } - name = p; - name.pop_back(); - auto l = name.size(); - if (l >= 3 && name[l-1] == 't' && name[l-2] == 'e' && - name[l-3] == 'R') { - isRet = true; - name.pop_back(); - name.pop_back(); - name.pop_back(); - } - inbrackets = q; + std::smatch match_obj; + if(!std::regex_search(line, match_obj, re)){ + throw std::logic_error("could not parse line"); + } + + threadName = match_obj[1]; + tid = std::stoi(match_obj[2]); + cpu = match_obj[3]; + startTime = std::stod(match_obj[4]); + duration = 0; + name = match_obj[6]; + inbrackets = match_obj[7]; + if (match_obj[9].length() > 0) { + isRet = true; + name.erase(name.end() - 3, name.end()); // remove Ret suffix form name } - free(s); } - bool empty() { - return name.empty(); - } + bool empty() { return name.empty(); } - string id() { - return to_string(tid) + name; - } + string id() { return to_string(tid) + name; } string pretty() { return to_string(duration) + " " + name + " " + to_string(startTime); @@ -77,31 +62,46 @@ struct Event { } }; -int main(int argc, char* argv[]) { - unordered_map table; - vector list; +// sample output: +// arangod 32636 [005] 16678249.324973: probe_arangod:insertLocalRet: (14a7f60 <- 14a78d6) +// process tid core timepoint scope:name frame +const regex Event::re( + R"_(\s*(\S+))_" // name 1 + R"_(\s+(\d+))_" // tid 2 + R"_(\s+\[(\d+)\])_" // cup 3 + R"_(\s+(\d+\.\d+):)_" // time 4 + R"_(\s+([^: ]+):([^: ]+):)_" // scope:func 5:6 + R"_(\s+\(([0-9a-f]+)(\s+<-\s+([0-9a-f]+))?\))_" // (start -> stop) 7 -> 9 + , + std::regex_constants::ECMAScript | std::regex_constants::optimize); + +int main(int /*argc*/, char* /*argv*/ []) { + unordered_map> table; + vector> list; string line; while (getline(cin, line)) { - Event* e = new Event(line); - if (!e->empty()) { - string id = e->id(); - if (!e->isRet) { + auto event = std::make_unique(line); + if (!event->empty()) { + string id = event->id(); + // insert to table if it is not a function return + if (!event->isRet) { auto it = table.find(id); if (it != table.end()) { cout << "Alarm, double event found:\n" << line << std::endl; } else { - table.insert(make_pair(id, e)); + table.insert(make_pair(id, std::move(event))); } + // update duration in table } else { auto it = table.find(id); if (it == table.end()) { cout << "Return for unknown event found:\n" << line << std::endl; } else { - Event* s = it->second; + unique_ptr ev = std::move(it->second); table.erase(it); - s->duration = e->startTime - s->startTime; - list.push_back(s); + ev->duration = event->startTime - ev->startTime; + list.push_back(std::move(ev)); } } } @@ -109,13 +109,11 @@ int main(int argc, char* argv[]) { cout << "Unreturned events:\n"; for (auto& p : table) { cout << p.second->pretty() << "\n"; - delete p.second; } - sort(list.begin(), list.end(), [](Event* a, Event* b) -> bool { - return *a < *b; - }); + sort(list.begin(), list.end(), + [](unique_ptrconst& a, unique_ptrconst& b) -> bool { return *a < *b; }); cout << "Events sorted by name and time:\n"; - for (auto* e : list) { + for (auto& e : list) { cout << e->pretty() << "\n"; } return 0; From 84c9b301ce6feba8fca19f6d02e99eb26ee5503a Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Mon, 29 Aug 2016 08:55:48 +0200 Subject: [PATCH 16/24] fixed issue #2023: added replicationFactor to docs --- .../Rest/Collections/JSF_post_api_collection.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md b/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md index 822a0a3521..ff4f206cf8 100644 --- a/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md +++ b/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md @@ -94,8 +94,19 @@ and the hash value is used to determine the target shard. **Note**: Values of shard key attributes cannot be changed once set. This option is meaningless in a single server setup. +@RESTBODYPARAM{replicationFactor,integer,optional,int64} +(The default is *1*): in a cluster, this attribute determines how many copies +of each shard are kept on different DBServers. The value 1 means that only one +copy (no synchronous replication) is kept. A value of k means that k-1 replicas +are kept. Any two copies reside on different DBServers. Replication between them is +synchronous, that is, every write operation to the "leader" copy will be replicated +to all "follower" replicas, before the write operation is reported successful. + +If a server fails, this is detected automatically and one of the servers holding +copies take over, usually without an error being reported. + @RESTDESCRIPTION -Creates an new collection with a given name. The request must contain an +Creates a new collection with a given name. The request must contain an object with the following attributes. From 4b5249dc5eba9288545e50c930026cc99c3b9aa4 Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Mon, 29 Aug 2016 09:34:23 +0200 Subject: [PATCH 17/24] issue #2022: double max allowed request body size, cap --batch-size value in arangoimp --- arangod/GeneralServer/HttpCommTask.cpp | 6 +++--- arangosh/Import/ImportFeature.cpp | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/arangod/GeneralServer/HttpCommTask.cpp b/arangod/GeneralServer/HttpCommTask.cpp index 0e37faa2bc..9546236694 100644 --- a/arangod/GeneralServer/HttpCommTask.cpp +++ b/arangod/GeneralServer/HttpCommTask.cpp @@ -47,9 +47,9 @@ class HttpResponse; namespace rest { class GeneralServer; -size_t const HttpCommTask::MaximalHeaderSize = 1 * 1024 * 1024; // 1 MB -size_t const HttpCommTask::MaximalBodySize = 512 * 1024 * 1024; // 512 MB -size_t const HttpCommTask::MaximalPipelineSize = 512 * 1024 * 1024; // 512 MB +size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024; // 1 MB +size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024; // 1024 MB +size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024; // 512 MB size_t const HttpCommTask::RunCompactEvery = 500; HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock, diff --git a/arangosh/Import/ImportFeature.cpp b/arangosh/Import/ImportFeature.cpp index 5ce571a610..b1fb92ef79 100644 --- a/arangosh/Import/ImportFeature.cpp +++ b/arangosh/Import/ImportFeature.cpp @@ -161,6 +161,16 @@ void ImportFeature::validateOptions( StringUtils::join(positionals, ", "); FATAL_ERROR_EXIT(); } + + static unsigned const MaxBatchSize = 768 * 1024 * 1024; + + if (_chunkSize > MaxBatchSize) { + // it's not sensible to raise the batch size beyond this value + // because the server has a built-in limit for the batch size too + // and will reject bigger HTTP request bodies + LOG(WARN) << "capping --batch-size value to " << MaxBatchSize; + _chunkSize = MaxBatchSize; + } } void ImportFeature::start() { From 524492c75d7ec00b957e36851e0756e01bf6eb27 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 29 Aug 2016 11:12:14 +0200 Subject: [PATCH 18/24] fixed cluster start --- scripts/startLocalCluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/startLocalCluster.sh b/scripts/startLocalCluster.sh index d0924bc501..fe83e3ed53 100755 --- a/scripts/startLocalCluster.sh +++ b/scripts/startLocalCluster.sh @@ -96,6 +96,7 @@ start() { PORT=$2 mkdir cluster/data$PORT echo Starting $TYPE on port $PORT + mkdir -p cluster/apps$PORT build/bin/arangod -c none \ --database.directory cluster/data$PORT \ --cluster.agency-endpoint tcp://127.0.0.1:$BASE \ From dc8ced3a9b9eeb933f5674d7f426f2ef4bbfc4ad Mon Sep 17 00:00:00 2001 From: Simran Brucherseifer Date: Mon, 29 Aug 2016 11:52:09 +0200 Subject: [PATCH 19/24] Fix comments --- arangod/GeneralServer/HttpCommTask.cpp | 4 ++-- lib/SimpleHttpClient/SimpleHttpResult.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arangod/GeneralServer/HttpCommTask.cpp b/arangod/GeneralServer/HttpCommTask.cpp index 9546236694..2bfd650266 100644 --- a/arangod/GeneralServer/HttpCommTask.cpp +++ b/arangod/GeneralServer/HttpCommTask.cpp @@ -47,9 +47,9 @@ class HttpResponse; namespace rest { class GeneralServer; -size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024; // 1 MB +size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024; // 2 MB size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024; // 1024 MB -size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024; // 512 MB +size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024; // 1024 MB size_t const HttpCommTask::RunCompactEvery = 500; HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock, diff --git a/lib/SimpleHttpClient/SimpleHttpResult.cpp b/lib/SimpleHttpClient/SimpleHttpResult.cpp index 0099965ecf..0be09c0188 100644 --- a/lib/SimpleHttpClient/SimpleHttpResult.cpp +++ b/lib/SimpleHttpClient/SimpleHttpResult.cpp @@ -162,7 +162,7 @@ void SimpleHttpResult::addHeaderField(char const* key, size_t keyLength, if (_returnCode == 204) { // HTTP 204 = No content. Assume we will have a content-length of 0. - // note that will value can be overridden later if the response has the content-length + // note that the value can be overridden later if the response has the content-length // header set to some other value setContentLength(0); } From dbd19cb48e4641157668c91f4f9d2e0869aba6be Mon Sep 17 00:00:00 2001 From: Jan Steemann Date: Mon, 29 Aug 2016 12:42:32 +0200 Subject: [PATCH 20/24] hexify corrupted markers --- arangod/VocBase/datafile.cpp | 64 ++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index 40e3ad6464..c72d501652 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -34,6 +34,7 @@ #include "VocBase/server.h" #include +#include // #define DEBUG_DATAFILE 1 @@ -756,7 +757,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { bool nextMarkerOk = false; if (size > 0) { - auto next = reinterpret_cast(marker) + size; + auto next = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); auto p = next; if (p < end) { @@ -783,7 +784,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { // there is a next marker auto nextMarker = reinterpret_cast(next); - + if (nextMarker->getType() != 0 && nextMarker->getSize() >= sizeof(TRI_df_marker_t) && next + nextMarker->getSize() <= end && @@ -810,10 +811,67 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); + LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' of size " + << datafile->_maximalSize << ", at position " << currentSize; + + LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker) + << "' and size " << size + << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); + + { + LOG(INFO) << "raw marker data following:"; + char const* p = reinterpret_cast(marker); + char const* e = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); + std::string line; + std::string raw; + size_t printed = 0; + while (p < e) { + // print offset + line.append("0x"); + uintptr_t offset = static_cast(p - datafile->_data); + for (size_t i = 0; i < 8; ++i) { + uint8_t c = static_cast((offset & (0xFFULL << 8 * (7 - i))) >> 8 * (7 - i)); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + } + + // print data + line.append(": "); + for (size_t i = 0; i < 16; ++i) { + if (p >= e) { + line.append(" "); + } else { + uint8_t c = static_cast(*p++); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + line.push_back(' '); + + raw.push_back((c < 32 || c >= 127) ? '.' : static_cast(c)); + + ++printed; + } + } + + LOG(INFO) << line << " " << raw; + line.clear(); + raw.clear(); + + if (printed >= 2048) { + LOG(INFO) << "(output truncated due to excessive length)"; + break; + } + } + } if (nextMarkerOk) { LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it"; + LOG(INFO) << "please restart the server with the parameter '--wal.ignore-logfile-errors true' to repair the marker"; } else { LOG(WARN) << "data directly following this marker cannot be analyzed"; } From abdb06a177aa613e7490afeaa2cf72edd988541b Mon Sep 17 00:00:00 2001 From: Jan Christoph Uhde Date: Fri, 26 Aug 2016 14:01:53 +0200 Subject: [PATCH 21/24] fix CMake so it finds the PythonInterpreter even with oder versions --- CMakeLists.txt | 2 +- cmake/FindPythonInterp.cmake | 165 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 cmake/FindPythonInterp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 164b455102..54ce226e3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ endif () set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # where to find CMAKE modules -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) +set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) # be verbose about flags used option(VERBOSE OFF) diff --git a/cmake/FindPythonInterp.cmake b/cmake/FindPythonInterp.cmake new file mode 100644 index 0000000000..e194185f22 --- /dev/null +++ b/cmake/FindPythonInterp.cmake @@ -0,0 +1,165 @@ +#.rst: +# FindPythonInterp +# ---------------- +# +# Find python interpreter +# +# This module finds if Python interpreter is installed and determines +# where the executables are. This code sets the following variables: +# +# :: +# +# PYTHONINTERP_FOUND - Was the Python executable found +# PYTHON_EXECUTABLE - path to the Python interpreter +# +# +# +# :: +# +# PYTHON_VERSION_STRING - Python version found e.g. 2.5.2 +# PYTHON_VERSION_MAJOR - Python major version found e.g. 2 +# PYTHON_VERSION_MINOR - Python minor version found e.g. 5 +# PYTHON_VERSION_PATCH - Python patch version found e.g. 2 +# +# +# +# The Python_ADDITIONAL_VERSIONS variable can be used to specify a list +# of version numbers that should be taken into account when searching +# for Python. You need to set this variable before calling +# find_package(PythonInterp). +# +# If calling both ``find_package(PythonInterp)`` and +# ``find_package(PythonLibs)``, call ``find_package(PythonInterp)`` first to +# get the currently active Python version by default with a consistent version +# of PYTHON_LIBRARIES. + +#============================================================================= +# Copyright 2005-2010 Kitware, Inc. +# Copyright 2011 Bjoern Ricks +# Copyright 2012 Rolf Eike Beer +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +unset(_Python_NAMES) + +set(_PYTHON1_VERSIONS 1.6 1.5) +set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0) +set(_PYTHON3_VERSIONS 3.6 3.5 3.4 3.3 3.2 3.1 3.0) + +if(PythonInterp_FIND_VERSION) + if(PythonInterp_FIND_VERSION_COUNT GREATER 1) + set(_PYTHON_FIND_MAJ_MIN "${PythonInterp_FIND_VERSION_MAJOR}.${PythonInterp_FIND_VERSION_MINOR}") + list(APPEND _Python_NAMES + python${_PYTHON_FIND_MAJ_MIN} + python${PythonInterp_FIND_VERSION_MAJOR}) + unset(_PYTHON_FIND_OTHER_VERSIONS) + if(NOT PythonInterp_FIND_VERSION_EXACT) + foreach(_PYTHON_V ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS}) + if(NOT _PYTHON_V VERSION_LESS _PYTHON_FIND_MAJ_MIN) + list(APPEND _PYTHON_FIND_OTHER_VERSIONS ${_PYTHON_V}) + endif() + endforeach() + endif() + unset(_PYTHON_FIND_MAJ_MIN) + else() + list(APPEND _Python_NAMES python${PythonInterp_FIND_VERSION_MAJOR}) + set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS}) + endif() +else() + set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON3_VERSIONS} ${_PYTHON2_VERSIONS} ${_PYTHON1_VERSIONS}) +endif() +find_program(PYTHON_EXECUTABLE NAMES ${_Python_NAMES}) + +# Set up the versions we know about, in the order we will search. Always add +# the user supplied additional versions to the front. +set(_Python_VERSIONS ${Python_ADDITIONAL_VERSIONS}) +# If FindPythonInterp has already found the major and minor version, +# insert that version next to get consistent versions of the interpreter and +# library. +if(DEFINED PYTHONLIBS_VERSION_STRING) + string(REPLACE "." ";" _PYTHONLIBS_VERSION "${PYTHONLIBS_VERSION_STRING}") + list(GET _PYTHONLIBS_VERSION 0 _PYTHONLIBS_VERSION_MAJOR) + list(GET _PYTHONLIBS_VERSION 1 _PYTHONLIBS_VERSION_MINOR) + list(APPEND _Python_VERSIONS ${_PYTHONLIBS_VERSION_MAJOR}.${_PYTHONLIBS_VERSION_MINOR}) +endif() +# Search for the current active python version first +list(APPEND _Python_VERSIONS ";") +list(APPEND _Python_VERSIONS ${_PYTHON_FIND_OTHER_VERSIONS}) + +unset(_PYTHON_FIND_OTHER_VERSIONS) +unset(_PYTHON1_VERSIONS) +unset(_PYTHON2_VERSIONS) +unset(_PYTHON3_VERSIONS) + +# Search for newest python version if python executable isn't found +if(NOT PYTHON_EXECUTABLE) + foreach(_CURRENT_VERSION IN LISTS _Python_VERSIONS) + set(_Python_NAMES python${_CURRENT_VERSION}) + if(WIN32) + list(APPEND _Python_NAMES python) + endif() + find_program(PYTHON_EXECUTABLE + NAMES ${_Python_NAMES} + PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath] + ) + endforeach() +endif() + +# determine python version string +if(PYTHON_EXECUTABLE) + execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c + "import sys; sys.stdout.write(';'.join([str(x) for x in sys.version_info[:3]]))" + OUTPUT_VARIABLE _VERSION + RESULT_VARIABLE _PYTHON_VERSION_RESULT + ERROR_QUIET) + if(NOT _PYTHON_VERSION_RESULT) + string(REPLACE ";" "." PYTHON_VERSION_STRING "${_VERSION}") + list(GET _VERSION 0 PYTHON_VERSION_MAJOR) + list(GET _VERSION 1 PYTHON_VERSION_MINOR) + list(GET _VERSION 2 PYTHON_VERSION_PATCH) + if(PYTHON_VERSION_PATCH EQUAL 0) + # it's called "Python 2.7", not "2.7.0" + string(REGEX REPLACE "\\.0$" "" PYTHON_VERSION_STRING "${PYTHON_VERSION_STRING}") + endif() + else() + # sys.version predates sys.version_info, so use that + execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.version)" + OUTPUT_VARIABLE _VERSION + RESULT_VARIABLE _PYTHON_VERSION_RESULT + ERROR_QUIET) + if(NOT _PYTHON_VERSION_RESULT) + string(REGEX REPLACE " .*" "" PYTHON_VERSION_STRING "${_VERSION}") + string(REGEX REPLACE "^([0-9]+)\\.[0-9]+.*" "\\1" PYTHON_VERSION_MAJOR "${PYTHON_VERSION_STRING}") + string(REGEX REPLACE "^[0-9]+\\.([0-9])+.*" "\\1" PYTHON_VERSION_MINOR "${PYTHON_VERSION_STRING}") + if(PYTHON_VERSION_STRING MATCHES "^[0-9]+\\.[0-9]+\\.([0-9]+)") + set(PYTHON_VERSION_PATCH "${CMAKE_MATCH_1}") + else() + set(PYTHON_VERSION_PATCH "0") + endif() + else() + # sys.version was first documented for Python 1.5, so assume + # this is older. + set(PYTHON_VERSION_STRING "1.4") + set(PYTHON_VERSION_MAJOR "1") + set(PYTHON_VERSION_MINOR "4") + set(PYTHON_VERSION_PATCH "0") + endif() + endif() + unset(_PYTHON_VERSION_RESULT) + unset(_VERSION) +endif() + +# handle the QUIETLY and REQUIRED arguments and set PYTHONINTERP_FOUND to TRUE if +# all listed variables are TRUE +include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonInterp REQUIRED_VARS PYTHON_EXECUTABLE VERSION_VAR PYTHON_VERSION_STRING) + +mark_as_advanced(PYTHON_EXECUTABLE) From 1be060bc9873c2116547464f01e82e536de6b53c Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 29 Aug 2016 16:17:14 +0200 Subject: [PATCH 22/24] only test for cleaning up 3rdparty if we're parametrized to do so. --- Installation/Jenkins/build.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index ffc462823b..17e66d48d4 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -101,16 +101,6 @@ if test -f last_compiled_version.sha; then fi COMPILE_MATTERS="3rdParty" -CLEAN_IT=1 - -if test -n "$LASTREV"; then - lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l` - - if test $lines -eq 0; then - echo "no relevant changes, no need for full recompile" - CLEAN_IT=0 - fi -fi # setup make options if test -z "${CXX}"; then @@ -193,6 +183,8 @@ case "$1" in ;; esac +CLEAN_IT=0 + while [ $# -gt 0 ]; do case "$1" in @@ -303,7 +295,11 @@ while [ $# -gt 0 ]; do TARGET_DIR=$1 shift ;; - + + --checkCleanBuild) + CLEAN_IT=1 + shift + ;; *) echo "Unknown option: $1" exit 1 @@ -311,6 +307,18 @@ while [ $# -gt 0 ]; do esac done + +if test -n "$LASTREV"; then + lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l` + + if test $lines -eq 0; then + echo "no relevant changes, no need for full recompile" + CLEAN_IT=0 + fi +fi + + + if [ "$GCC5" == 1 ]; then CC=/usr/bin/gcc-5 CXX=/usr/bin/g++-5 From 0e36f95f1a7023c9012ea75e92481c01abb2315d Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 29 Aug 2016 16:34:17 +0200 Subject: [PATCH 23/24] fix effect that on second install we will complain about mismatching passwords --- Installation/debian/postinst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Installation/debian/postinst b/Installation/debian/postinst index 63297fffbb..152e3d3424 100755 --- a/Installation/debian/postinst +++ b/Installation/debian/postinst @@ -15,8 +15,9 @@ if [ "$1" = "configure" -a -z "$2" ]; then /usr/sbin/arango-init-database \ --uid arangodb --gid arangodb || true fi - + db_set arangodb3/password_again "" db_set arangodb3/password "" + db_go fi # check if we should upgrade the database directory From c53428ce893e1684701d8be23a894b221ad4969d Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 29 Aug 2016 17:13:33 +0200 Subject: [PATCH 24/24] minor fixes --- arangod/Agency/AgentConfiguration.cpp | 4 ++-- arangod/RestServer/BootstrapFeature.cpp | 2 +- js/server/modules/@arangodb/foxx/queues/index.js | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arangod/Agency/AgentConfiguration.cpp b/arangod/Agency/AgentConfiguration.cpp index a1a9dff00d..572477204c 100644 --- a/arangod/Agency/AgentConfiguration.cpp +++ b/arangod/Agency/AgentConfiguration.cpp @@ -476,7 +476,7 @@ bool config_t::merge(VPackSlice const& conf) { ss << "Min RAFT interval: "; if (_minPing == 0) { // Command line beats persistence if (conf.hasKey(minPingStr)) { - _minPing = conf.get(minPingStr).getUInt(); + _minPing = conf.get(minPingStr).getDouble(); ss << _minPing << " (persisted)"; } else { _minPing = 0.5; @@ -491,7 +491,7 @@ bool config_t::merge(VPackSlice const& conf) { ss << "Max RAFT interval: "; if (_maxPing == 0) { // Command line beats persistence if (conf.hasKey(maxPingStr)) { - _maxPing = conf.get(maxPingStr).getUInt(); + _maxPing = conf.get(maxPingStr).getDouble(); ss << _maxPing << " (persisted)"; } else { _maxPing = 2.5; diff --git a/arangod/RestServer/BootstrapFeature.cpp b/arangod/RestServer/BootstrapFeature.cpp index de7aa1b091..95f248ccbc 100644 --- a/arangod/RestServer/BootstrapFeature.cpp +++ b/arangod/RestServer/BootstrapFeature.cpp @@ -139,7 +139,7 @@ void BootstrapFeature::start() { auto vocbase = DatabaseFeature::DATABASE->vocbase(); auto ss = ServerState::instance(); - if (!ss->isRunningInCluster() && !ss->isAgent()) { + if (!ss->isRunningInCluster()) { LOG_TOPIC(DEBUG, Logger::STARTUP) << "Running server/server.js"; V8DealerFeature::DEALER->loadJavascript(vocbase, "server/server.js"); } else if (ss->isCoordinator()) { diff --git a/js/server/modules/@arangodb/foxx/queues/index.js b/js/server/modules/@arangodb/foxx/queues/index.js index 659e920ced..7bb89ee767 100644 --- a/js/server/modules/@arangodb/foxx/queues/index.js +++ b/js/server/modules/@arangodb/foxx/queues/index.js @@ -24,6 +24,7 @@ // ////////////////////////////////////////////////////////////////////////////// const isCluster = require("@arangodb/cluster").isCluster(); +const isAgent = global.ArangoAgent.enabled(); var _ = require('lodash'); var flatten = require('internal').flatten; @@ -231,7 +232,7 @@ function asNumber (num) { } function updateQueueDelayClusterAware() { - if (isCluster) { + if (isCluster && !isAgent) { global.ArangoAgency.set('Current/FoxxmasterQueueupdate', true); } updateQueueDelay();