diff --git a/CMakeLists.txt b/CMakeLists.txt index dca7baf41d..54ce226e3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ endif () set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # where to find CMAKE modules -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake) +set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) # be verbose about flags used option(VERBOSE OFF) @@ -708,7 +708,7 @@ if (NOT USE_BOOST_UNITTESTS) message(STATUS "BOOST unit-tests are disabled") endif () -include_directories(${Boost_INCLUDE_DIR}) +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) add_definitions(-DARANGODB_BOOST_VERSION=\"${Boost_VERSION}\") ################################################################################ diff --git a/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md b/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md index 822a0a3521..ff4f206cf8 100644 --- a/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md +++ b/Documentation/DocuBlocks/Rest/Collections/JSF_post_api_collection.md @@ -94,8 +94,19 @@ and the hash value is used to determine the target shard. **Note**: Values of shard key attributes cannot be changed once set. This option is meaningless in a single server setup. +@RESTBODYPARAM{replicationFactor,integer,optional,int64} +(The default is *1*): in a cluster, this attribute determines how many copies +of each shard are kept on different DBServers. The value 1 means that only one +copy (no synchronous replication) is kept. A value of k means that k-1 replicas +are kept. Any two copies reside on different DBServers. Replication between them is +synchronous, that is, every write operation to the "leader" copy will be replicated +to all "follower" replicas, before the write operation is reported successful. + +If a server fails, this is detected automatically and one of the servers holding +copies take over, usually without an error being reported. + @RESTDESCRIPTION -Creates an new collection with a given name. The request must contain an +Creates a new collection with a given name. The request must contain an object with the following attributes. diff --git a/arangod/Agency/v8-agency.cpp b/arangod/Agency/v8-agency.cpp index 8f6437d9ec..3ede27d769 100644 --- a/arangod/Agency/v8-agency.cpp +++ b/arangod/Agency/v8-agency.cpp @@ -41,39 +41,149 @@ using namespace arangodb::application_features; using namespace arangodb::basics; using namespace arangodb::consensus; -static void JS_LeadingVulpes(v8::FunctionCallbackInfo const& args) { +static void JS_EnabledAgent(v8::FunctionCallbackInfo const& args) { - v8::Isolate* isolate = args.GetIsolate(); + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + try { + ApplicationServer::getEnabledFeature("Agency"); + TRI_V8_RETURN_TRUE(); + } catch (std::exception const& e) { + TRI_V8_RETURN_FALSE(); + } + + TRI_V8_TRY_CATCH_END + +} + +static void JS_LeadingAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + Agent* agent = nullptr; try { AgencyFeature* feature = - ApplicationServer::getEnabledFeature("AgencyFeature"); + ApplicationServer::getEnabledFeature("Agency"); agent = feature->agent(); + + } catch (std::exception const& e) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, + std::string("couldn't access agency feature: ") + e.what()); + } + + v8::Handle r = v8::Object::New(isolate); + + r->Set(TRI_V8_ASCII_STRING("leading"), + v8::Boolean::New(isolate, agent->leading())); + + TRI_V8_RETURN(r); + TRI_V8_TRY_CATCH_END + +} + +static void JS_ReadAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + Agent* agent = nullptr; + try { + AgencyFeature* feature = + ApplicationServer::getEnabledFeature("Agency"); + agent = feature->agent(); + + } catch (std::exception const& e) { + TRI_V8_THROW_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, + std::string("couldn't access agency feature: ") + e.what()); + } + + query_t query = std::make_shared(); + int res = TRI_V8ToVPack(isolate, *query, args[0], false); + + if (res != TRI_ERROR_NO_ERROR) { + TRI_V8_THROW_EXCEPTION(res); + } + + read_ret_t ret = agent->read(query); + + if (ret.accepted) { // Leading + TRI_V8_RETURN(TRI_VPackToV8(isolate, ret.result->slice())); + } else { // Not leading + TRI_V8_RETURN_FALSE(); + } + + TRI_V8_TRY_CATCH_END + +} + + +static void JS_WriteAgent(v8::FunctionCallbackInfo const& args) { + + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + Agent* agent = nullptr; + try { + AgencyFeature* feature = + ApplicationServer::getEnabledFeature("Agency"); + agent = feature->agent(); + } catch (std::exception const& e) { TRI_V8_THROW_EXCEPTION_MESSAGE( TRI_ERROR_INTERNAL, std::string("couldn't access agency feature: ") + e.what()); } - v8::Handle r = v8::Object::New(isolate); + query_t query = std::make_shared(); + int res = TRI_V8ToVPack(isolate, *query, args[0], false); + if (res != TRI_ERROR_NO_ERROR) { + TRI_V8_THROW_EXCEPTION(res); + } - r->Set(TRI_V8_ASCII_STRING("leading"), - v8::Boolean::New(isolate, agent->leading())); - - TRI_V8_RETURN(r); + write_ret_t ret = agent->write(query); + if (ret.accepted) { // Leading + + size_t errors = 0; + Builder body; + body.openObject(); + body.add("results", VPackValue(VPackValueType::Array)); + for (auto const& index : ret.indices) { + body.add(VPackValue(index)); + if (index == 0) { + errors++; + } + } + body.close(); body.close(); + + // Wait for commit of highest except if it is 0? + arangodb::consensus::index_t max_index = 0; + try { + max_index = + *std::max_element(ret.indices.begin(), ret.indices.end()); + } catch (std::exception const& e) { + LOG_TOPIC(WARN, Logger::AGENCY) + << e.what() << " " << __FILE__ << __LINE__; + } + + if (max_index > 0) { + agent->waitFor(max_index); + } + + TRI_V8_RETURN(TRI_VPackToV8(isolate, body.slice())); + } else { // Not leading + TRI_V8_RETURN_FALSE(); + } -} - -static void JS_ReadVulpes(v8::FunctionCallbackInfo const& args) { - -} - -static void JS_WriteVulpes(v8::FunctionCallbackInfo const& args) { - + TRI_V8_TRY_CATCH_END + } @@ -89,30 +199,32 @@ void TRI_InitV8Agency(v8::Isolate* isolate, v8::Handle context) { // ........................................................................... ft = v8::FunctionTemplate::New(isolate); - ft->SetClassName(TRI_V8_ASCII_STRING("ArangoVulpes")); + ft->SetClassName(TRI_V8_ASCII_STRING("ArangoAgent")); rt = ft->InstanceTemplate(); rt->SetInternalFieldCount(2); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("leading"), JS_LeadingVulpes); + isolate, rt, TRI_V8_ASCII_STRING("enabled"), JS_EnabledAgent); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("read"), JS_ReadVulpes); + isolate, rt, TRI_V8_ASCII_STRING("leading"), JS_LeadingAgent); TRI_AddMethodVocbase( - isolate, rt, TRI_V8_ASCII_STRING("write"), JS_WriteVulpes); + isolate, rt, TRI_V8_ASCII_STRING("read"), JS_ReadAgent); + TRI_AddMethodVocbase( + isolate, rt, TRI_V8_ASCII_STRING("write"), JS_WriteAgent); - v8g->VulpesTempl.Reset(isolate, rt); - ft->SetClassName(TRI_V8_ASCII_STRING("ArangoVuplesCtor")); + v8g->AgentTempl.Reset(isolate, rt); + ft->SetClassName(TRI_V8_ASCII_STRING("ArangoAgentCtor")); TRI_AddGlobalFunctionVocbase( - isolate, context, TRI_V8_ASCII_STRING("ArangoVuplesCtor"), + isolate, context, TRI_V8_ASCII_STRING("ArangoAgentCtor"), ft->GetFunction(), true); // register the global object v8::Handle aa = rt->NewInstance(); if (!aa.IsEmpty()) { TRI_AddGlobalVariableVocbase( - isolate, context, TRI_V8_ASCII_STRING("ArangoVuples"), aa); + isolate, context, TRI_V8_ASCII_STRING("ArangoAgent"), aa); } } diff --git a/arangod/GeneralServer/HttpCommTask.cpp b/arangod/GeneralServer/HttpCommTask.cpp index 0e37faa2bc..2bfd650266 100644 --- a/arangod/GeneralServer/HttpCommTask.cpp +++ b/arangod/GeneralServer/HttpCommTask.cpp @@ -47,9 +47,9 @@ class HttpResponse; namespace rest { class GeneralServer; -size_t const HttpCommTask::MaximalHeaderSize = 1 * 1024 * 1024; // 1 MB -size_t const HttpCommTask::MaximalBodySize = 512 * 1024 * 1024; // 512 MB -size_t const HttpCommTask::MaximalPipelineSize = 512 * 1024 * 1024; // 512 MB +size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024; // 2 MB +size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024; // 1024 MB +size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024; // 1024 MB size_t const HttpCommTask::RunCompactEvery = 500; HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock, diff --git a/arangod/RestServer/DatabaseFeature.cpp b/arangod/RestServer/DatabaseFeature.cpp index 4a722e038f..ff55356c32 100644 --- a/arangod/RestServer/DatabaseFeature.cpp +++ b/arangod/RestServer/DatabaseFeature.cpp @@ -24,6 +24,7 @@ #include "Basics/StringUtils.h" #include "Basics/ArangoGlobalContext.h" +#include "Agency/v8-agency.h" #include "Cluster/ServerState.h" #include "Cluster/v8-cluster.h" #include "GeneralServer/GeneralServerFeature.h" @@ -219,6 +220,7 @@ void DatabaseFeature::updateContexts() { i); TRI_InitV8Queries(isolate, context); TRI_InitV8Cluster(isolate, context); + TRI_InitV8Agency(isolate, context); }, vocbase); } diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index 40e3ad6464..c72d501652 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -34,6 +34,7 @@ #include "VocBase/server.h" #include +#include // #define DEBUG_DATAFILE 1 @@ -756,7 +757,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { bool nextMarkerOk = false; if (size > 0) { - auto next = reinterpret_cast(marker) + size; + auto next = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); auto p = next; if (p < end) { @@ -783,7 +784,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { // there is a next marker auto nextMarker = reinterpret_cast(next); - + if (nextMarker->getType() != 0 && nextMarker->getSize() >= sizeof(TRI_df_marker_t) && next + nextMarker->getSize() <= end && @@ -810,10 +811,67 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) { datafile->_next = datafile->_data + datafile->_currentSize; datafile->_state = TRI_DF_STATE_OPEN_ERROR; - LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); + LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' of size " + << datafile->_maximalSize << ", at position " << currentSize; + + LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker) + << "' and size " << size + << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); + + { + LOG(INFO) << "raw marker data following:"; + char const* p = reinterpret_cast(marker); + char const* e = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); + std::string line; + std::string raw; + size_t printed = 0; + while (p < e) { + // print offset + line.append("0x"); + uintptr_t offset = static_cast(p - datafile->_data); + for (size_t i = 0; i < 8; ++i) { + uint8_t c = static_cast((offset & (0xFFULL << 8 * (7 - i))) >> 8 * (7 - i)); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + } + + // print data + line.append(": "); + for (size_t i = 0; i < 16; ++i) { + if (p >= e) { + line.append(" "); + } else { + uint8_t c = static_cast(*p++); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + line.push_back(' '); + + raw.push_back((c < 32 || c >= 127) ? '.' : static_cast(c)); + + ++printed; + } + } + + LOG(INFO) << line << " " << raw; + line.clear(); + raw.clear(); + + if (printed >= 2048) { + LOG(INFO) << "(output truncated due to excessive length)"; + break; + } + } + } if (nextMarkerOk) { LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it"; + LOG(INFO) << "please restart the server with the parameter '--wal.ignore-logfile-errors true' to repair the marker"; } else { LOG(WARN) << "data directly following this marker cannot be analyzed"; } diff --git a/arangosh/Import/ImportFeature.cpp b/arangosh/Import/ImportFeature.cpp index 5ce571a610..b1fb92ef79 100644 --- a/arangosh/Import/ImportFeature.cpp +++ b/arangosh/Import/ImportFeature.cpp @@ -161,6 +161,16 @@ void ImportFeature::validateOptions( StringUtils::join(positionals, ", "); FATAL_ERROR_EXIT(); } + + static unsigned const MaxBatchSize = 768 * 1024 * 1024; + + if (_chunkSize > MaxBatchSize) { + // it's not sensible to raise the batch size beyond this value + // because the server has a built-in limit for the batch size too + // and will reject bigger HTTP request bodies + LOG(WARN) << "capping --batch-size value to " << MaxBatchSize; + _chunkSize = MaxBatchSize; + } } void ImportFeature::start() { diff --git a/cmake/FindPythonInterp.cmake b/cmake/FindPythonInterp.cmake new file mode 100644 index 0000000000..e194185f22 --- /dev/null +++ b/cmake/FindPythonInterp.cmake @@ -0,0 +1,165 @@ +#.rst: +# FindPythonInterp +# ---------------- +# +# Find python interpreter +# +# This module finds if Python interpreter is installed and determines +# where the executables are. This code sets the following variables: +# +# :: +# +# PYTHONINTERP_FOUND - Was the Python executable found +# PYTHON_EXECUTABLE - path to the Python interpreter +# +# +# +# :: +# +# PYTHON_VERSION_STRING - Python version found e.g. 2.5.2 +# PYTHON_VERSION_MAJOR - Python major version found e.g. 2 +# PYTHON_VERSION_MINOR - Python minor version found e.g. 5 +# PYTHON_VERSION_PATCH - Python patch version found e.g. 2 +# +# +# +# The Python_ADDITIONAL_VERSIONS variable can be used to specify a list +# of version numbers that should be taken into account when searching +# for Python. You need to set this variable before calling +# find_package(PythonInterp). +# +# If calling both ``find_package(PythonInterp)`` and +# ``find_package(PythonLibs)``, call ``find_package(PythonInterp)`` first to +# get the currently active Python version by default with a consistent version +# of PYTHON_LIBRARIES. + +#============================================================================= +# Copyright 2005-2010 Kitware, Inc. +# Copyright 2011 Bjoern Ricks +# Copyright 2012 Rolf Eike Beer +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +unset(_Python_NAMES) + +set(_PYTHON1_VERSIONS 1.6 1.5) +set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0) +set(_PYTHON3_VERSIONS 3.6 3.5 3.4 3.3 3.2 3.1 3.0) + +if(PythonInterp_FIND_VERSION) + if(PythonInterp_FIND_VERSION_COUNT GREATER 1) + set(_PYTHON_FIND_MAJ_MIN "${PythonInterp_FIND_VERSION_MAJOR}.${PythonInterp_FIND_VERSION_MINOR}") + list(APPEND _Python_NAMES + python${_PYTHON_FIND_MAJ_MIN} + python${PythonInterp_FIND_VERSION_MAJOR}) + unset(_PYTHON_FIND_OTHER_VERSIONS) + if(NOT PythonInterp_FIND_VERSION_EXACT) + foreach(_PYTHON_V ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS}) + if(NOT _PYTHON_V VERSION_LESS _PYTHON_FIND_MAJ_MIN) + list(APPEND _PYTHON_FIND_OTHER_VERSIONS ${_PYTHON_V}) + endif() + endforeach() + endif() + unset(_PYTHON_FIND_MAJ_MIN) + else() + list(APPEND _Python_NAMES python${PythonInterp_FIND_VERSION_MAJOR}) + set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS}) + endif() +else() + set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON3_VERSIONS} ${_PYTHON2_VERSIONS} ${_PYTHON1_VERSIONS}) +endif() +find_program(PYTHON_EXECUTABLE NAMES ${_Python_NAMES}) + +# Set up the versions we know about, in the order we will search. Always add +# the user supplied additional versions to the front. +set(_Python_VERSIONS ${Python_ADDITIONAL_VERSIONS}) +# If FindPythonInterp has already found the major and minor version, +# insert that version next to get consistent versions of the interpreter and +# library. +if(DEFINED PYTHONLIBS_VERSION_STRING) + string(REPLACE "." ";" _PYTHONLIBS_VERSION "${PYTHONLIBS_VERSION_STRING}") + list(GET _PYTHONLIBS_VERSION 0 _PYTHONLIBS_VERSION_MAJOR) + list(GET _PYTHONLIBS_VERSION 1 _PYTHONLIBS_VERSION_MINOR) + list(APPEND _Python_VERSIONS ${_PYTHONLIBS_VERSION_MAJOR}.${_PYTHONLIBS_VERSION_MINOR}) +endif() +# Search for the current active python version first +list(APPEND _Python_VERSIONS ";") +list(APPEND _Python_VERSIONS ${_PYTHON_FIND_OTHER_VERSIONS}) + +unset(_PYTHON_FIND_OTHER_VERSIONS) +unset(_PYTHON1_VERSIONS) +unset(_PYTHON2_VERSIONS) +unset(_PYTHON3_VERSIONS) + +# Search for newest python version if python executable isn't found +if(NOT PYTHON_EXECUTABLE) + foreach(_CURRENT_VERSION IN LISTS _Python_VERSIONS) + set(_Python_NAMES python${_CURRENT_VERSION}) + if(WIN32) + list(APPEND _Python_NAMES python) + endif() + find_program(PYTHON_EXECUTABLE + NAMES ${_Python_NAMES} + PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath] + ) + endforeach() +endif() + +# determine python version string +if(PYTHON_EXECUTABLE) + execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c + "import sys; sys.stdout.write(';'.join([str(x) for x in sys.version_info[:3]]))" + OUTPUT_VARIABLE _VERSION + RESULT_VARIABLE _PYTHON_VERSION_RESULT + ERROR_QUIET) + if(NOT _PYTHON_VERSION_RESULT) + string(REPLACE ";" "." PYTHON_VERSION_STRING "${_VERSION}") + list(GET _VERSION 0 PYTHON_VERSION_MAJOR) + list(GET _VERSION 1 PYTHON_VERSION_MINOR) + list(GET _VERSION 2 PYTHON_VERSION_PATCH) + if(PYTHON_VERSION_PATCH EQUAL 0) + # it's called "Python 2.7", not "2.7.0" + string(REGEX REPLACE "\\.0$" "" PYTHON_VERSION_STRING "${PYTHON_VERSION_STRING}") + endif() + else() + # sys.version predates sys.version_info, so use that + execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.version)" + OUTPUT_VARIABLE _VERSION + RESULT_VARIABLE _PYTHON_VERSION_RESULT + ERROR_QUIET) + if(NOT _PYTHON_VERSION_RESULT) + string(REGEX REPLACE " .*" "" PYTHON_VERSION_STRING "${_VERSION}") + string(REGEX REPLACE "^([0-9]+)\\.[0-9]+.*" "\\1" PYTHON_VERSION_MAJOR "${PYTHON_VERSION_STRING}") + string(REGEX REPLACE "^[0-9]+\\.([0-9])+.*" "\\1" PYTHON_VERSION_MINOR "${PYTHON_VERSION_STRING}") + if(PYTHON_VERSION_STRING MATCHES "^[0-9]+\\.[0-9]+\\.([0-9]+)") + set(PYTHON_VERSION_PATCH "${CMAKE_MATCH_1}") + else() + set(PYTHON_VERSION_PATCH "0") + endif() + else() + # sys.version was first documented for Python 1.5, so assume + # this is older. + set(PYTHON_VERSION_STRING "1.4") + set(PYTHON_VERSION_MAJOR "1") + set(PYTHON_VERSION_MINOR "4") + set(PYTHON_VERSION_PATCH "0") + endif() + endif() + unset(_PYTHON_VERSION_RESULT) + unset(_VERSION) +endif() + +# handle the QUIETLY and REQUIRED arguments and set PYTHONINTERP_FOUND to TRUE if +# all listed variables are TRUE +include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonInterp REQUIRED_VARS PYTHON_EXECUTABLE VERSION_VAR PYTHON_VERSION_STRING) + +mark_as_advanced(PYTHON_EXECUTABLE) diff --git a/lib/Basics/files.cpp b/lib/Basics/files.cpp index 5104bc5b79..5ca44696f4 100644 --- a/lib/Basics/files.cpp +++ b/lib/Basics/files.cpp @@ -1708,18 +1708,18 @@ std::string TRI_GetInstallRoot(std::string const& binaryPath, } size_t bpLength = binaryPath.length(); - const char *pbPath = binaryPath.c_str(); + char const* pbPath = binaryPath.c_str(); if (pbPath[bpLength - 1] == TRI_DIR_SEPARATOR_CHAR) { - bpLength --; + --bpLength; } if (ibpLength > bpLength) { return TRI_DIR_SEPARATOR_STR; } - for (int i = 1; i < ibpLength; i ++) { - if (pbPath[bpLength -i] != installBinaryPath[ibpLength - i]) { + for (size_t i = 1; i < ibpLength; ++i) { + if (pbPath[bpLength - i] != installBinaryPath[ibpLength - i]) { return TRI_DIR_SEPARATOR_STR; } } diff --git a/lib/SimpleHttpClient/SimpleHttpResult.cpp b/lib/SimpleHttpClient/SimpleHttpResult.cpp index 0099965ecf..0be09c0188 100644 --- a/lib/SimpleHttpClient/SimpleHttpResult.cpp +++ b/lib/SimpleHttpClient/SimpleHttpResult.cpp @@ -162,7 +162,7 @@ void SimpleHttpResult::addHeaderField(char const* key, size_t keyLength, if (_returnCode == 204) { // HTTP 204 = No content. Assume we will have a content-length of 0. - // note that will value can be overridden later if the response has the content-length + // note that the value can be overridden later if the response has the content-length // header set to some other value setContentLength(0); } diff --git a/lib/V8/v8-globals.cpp b/lib/V8/v8-globals.cpp index 46bee5e46d..63a4f0ea6b 100644 --- a/lib/V8/v8-globals.cpp +++ b/lib/V8/v8-globals.cpp @@ -28,6 +28,7 @@ TRI_v8_global_s::TRI_v8_global_s(v8::Isolate* isolate) JSVPack(), AgencyTempl(), + AgentTempl(), ClusterInfoTempl(), ServerStateTempl(), ClusterCommTempl(), @@ -35,7 +36,6 @@ TRI_v8_global_s::TRI_v8_global_s(v8::Isolate* isolate) VPackTempl(), VocbaseColTempl(), VocbaseTempl(), - VulpesTempl(), BufferTempl(), diff --git a/lib/V8/v8-globals.h b/lib/V8/v8-globals.h index 77ea333a99..f850f3c465 100644 --- a/lib/V8/v8-globals.h +++ b/lib/V8/v8-globals.h @@ -475,6 +475,12 @@ typedef struct TRI_v8_global_s { v8::Persistent AgencyTempl; + ////////////////////////////////////////////////////////////////////////////// + /// @brief local agent template + ////////////////////////////////////////////////////////////////////////////// + + v8::Persistent AgentTempl; + ////////////////////////////////////////////////////////////////////////////// /// @brief clusterinfo template ////////////////////////////////////////////////////////////////////////////// @@ -517,12 +523,6 @@ typedef struct TRI_v8_global_s { v8::Persistent VocbaseTempl; - ////////////////////////////////////////////////////////////////////////////// - /// @brief vulpes template - ////////////////////////////////////////////////////////////////////////////// - - v8::Persistent VulpesTempl; - ////////////////////////////////////////////////////////////////////////////// /// @brief TRI_vocbase_t template ////////////////////////////////////////////////////////////////////////////// diff --git a/scripts/perfanalysis.cpp b/scripts/perfanalysis.cpp index 631fd5372f..01a635b561 100644 --- a/scripts/perfanalysis.cpp +++ b/scripts/perfanalysis.cpp @@ -1,17 +1,20 @@ // Compile with // g++ perfanalysis.cpp -o perfanalyis -std=c++11 -Wall -O3 -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include using namespace std; struct Event { + static const regex re; string threadName; int tid; string cpu; @@ -22,45 +25,27 @@ struct Event { bool isRet; Event(string& line) : isRet(false) { - char* s = strdup(line.c_str()); - char* p = strtok(s, " "); - char* q; - if (p != nullptr) { - threadName = p; - p = strtok(nullptr, " "); - tid = strtol(p, nullptr, 10); - p = strtok(nullptr, " "); - cpu = p; - p = strtok(nullptr, " "); - startTime = strtod(p, nullptr); - p = strtok(nullptr, " "); - q = strtok(nullptr, " "); - if (strcmp(q, "cs:") == 0) { - free(s); - return; - } - name = p; - name.pop_back(); - auto l = name.size(); - if (l >= 3 && name[l-1] == 't' && name[l-2] == 'e' && - name[l-3] == 'R') { - isRet = true; - name.pop_back(); - name.pop_back(); - name.pop_back(); - } - inbrackets = q; + std::smatch match_obj; + if(!std::regex_search(line, match_obj, re)){ + throw std::logic_error("could not parse line"); + } + + threadName = match_obj[1]; + tid = std::stoi(match_obj[2]); + cpu = match_obj[3]; + startTime = std::stod(match_obj[4]); + duration = 0; + name = match_obj[6]; + inbrackets = match_obj[7]; + if (match_obj[9].length() > 0) { + isRet = true; + name.erase(name.end() - 3, name.end()); // remove Ret suffix form name } - free(s); } - bool empty() { - return name.empty(); - } + bool empty() { return name.empty(); } - string id() { - return to_string(tid) + name; - } + string id() { return to_string(tid) + name; } string pretty() { return to_string(duration) + " " + name + " " + to_string(startTime); @@ -77,31 +62,46 @@ struct Event { } }; -int main(int argc, char* argv[]) { - unordered_map table; - vector list; +// sample output: +// arangod 32636 [005] 16678249.324973: probe_arangod:insertLocalRet: (14a7f60 <- 14a78d6) +// process tid core timepoint scope:name frame +const regex Event::re( + R"_(\s*(\S+))_" // name 1 + R"_(\s+(\d+))_" // tid 2 + R"_(\s+\[(\d+)\])_" // cup 3 + R"_(\s+(\d+\.\d+):)_" // time 4 + R"_(\s+([^: ]+):([^: ]+):)_" // scope:func 5:6 + R"_(\s+\(([0-9a-f]+)(\s+<-\s+([0-9a-f]+))?\))_" // (start -> stop) 7 -> 9 + , + std::regex_constants::ECMAScript | std::regex_constants::optimize); + +int main(int /*argc*/, char* /*argv*/ []) { + unordered_map> table; + vector> list; string line; while (getline(cin, line)) { - Event* e = new Event(line); - if (!e->empty()) { - string id = e->id(); - if (!e->isRet) { + auto event = std::make_unique(line); + if (!event->empty()) { + string id = event->id(); + // insert to table if it is not a function return + if (!event->isRet) { auto it = table.find(id); if (it != table.end()) { cout << "Alarm, double event found:\n" << line << std::endl; } else { - table.insert(make_pair(id, e)); + table.insert(make_pair(id, std::move(event))); } + // update duration in table } else { auto it = table.find(id); if (it == table.end()) { cout << "Return for unknown event found:\n" << line << std::endl; } else { - Event* s = it->second; + unique_ptr ev = std::move(it->second); table.erase(it); - s->duration = e->startTime - s->startTime; - list.push_back(s); + ev->duration = event->startTime - ev->startTime; + list.push_back(std::move(ev)); } } } @@ -109,13 +109,11 @@ int main(int argc, char* argv[]) { cout << "Unreturned events:\n"; for (auto& p : table) { cout << p.second->pretty() << "\n"; - delete p.second; } - sort(list.begin(), list.end(), [](Event* a, Event* b) -> bool { - return *a < *b; - }); + sort(list.begin(), list.end(), + [](unique_ptrconst& a, unique_ptrconst& b) -> bool { return *a < *b; }); cout << "Events sorted by name and time:\n"; - for (auto* e : list) { + for (auto& e : list) { cout << e->pretty() << "\n"; } return 0; diff --git a/scripts/setupPerfEvents.sh b/scripts/setupPerfEvents.sh index fea79be017..36af7a012f 100755 --- a/scripts/setupPerfEvents.sh +++ b/scripts/setupPerfEvents.sh @@ -6,60 +6,78 @@ # # This script sets up performance monitoring events to measure single # document operations. Run this script with sudo when the ArangoDB -# process is already running. Then do +# process is already running: +# +# ./setupPerfEvents.sh +# +# Now you are able to recrod the event with: +# # sudo perf record -e "probe_arangod:*" -aR sleep 60 -# (to sample for 60 seconds). A file "perf.data" is written to the -# current directory. -# Dump the events in this file with +# +# The above command will get sample data for 60 seconds. A file "perf.data" is +# written to the current directory. Dump the events in this file with: +# # sudo perf script > perf.history +# # This logs the times when individual threads hit the events. # Use the program perfanalyis.cpp in this directory in the following way: +# # sudo ./perfanalyis < perf.history > perf.statistics -# This will group enter and exit events of functions together, compute -# the time spent and sort by function. -# Remove all events with +# +# This will group enter and exit events of functions together, compute the time +# spent and sort by function. When finised remove all events with: +# # sudo perf probe -d "probe_arangod:*" -# List events with +# +# List events with: +# # sudo perf probe -l +# +# -ARANGOD_EXECUTABLE=build/bin/arangod -perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" +main(){ + local ARANGOD_EXECUTABLE=${1-build/bin/arangod} -echo Adding events, this takes a few seconds... + #delete all existing events + perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*" + + echo "Adding events, this takes a few seconds..." + + echo "Single document operations..." + addEvent insertLocal + addEvent removeLocal + addEvent modifyLocal + addEvent documentLocal + + echo "Single document operations on coordinator..." + addEvent insertCoordinator + addEvent removeCoordinator + addEvent updateCoordinator + addEvent replaceCoordinator + addEvent documentCoordinator + + echo "work method in HttpServerJob" + addEvent workHttpServerJob work@HttpServerJob.cpp + + echo "work method in RestDocumentHandler" + addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp + addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp + addEvent handleRequest handleRequest@HttpServer.cpp + addEvent handleWrite handleWrite@SocketTask.cpp + + addEvent tcp_sendmsg + addEvent tcp_recvmsg + + echo Done. +} addEvent() { - x=$1 - y=$2 - if [ "x$y" == "x" ] ; then - y=$x - fi - echo $x - perf probe -x $ARANGOD_EXECUTABLE -a $x=$y 2> /dev/null - perf probe -x $ARANGOD_EXECUTABLE -a ${x}Ret=$y%return 2> /dev/null + local name="$1" + local func="${2-"${name}"}" + + echo "setting up $name for function: $func" + perf probe -x $ARANGOD_EXECUTABLE -a $name=$func 2> /dev/null #enter function + perf probe -x $ARANGOD_EXECUTABLE -a ${name}Ret=$func%return 2> /dev/null #return form function } -echo Single document operations... -addEvent insertLocal -addEvent removeLocal -addEvent modifyLocal -addEvent documentLocal -echo Single document operations on coordinator... -addEvent insertCoordinator -addEvent removeCoordinator -addEvent updateCoordinator -addEvent replaceCoordinator -addEvent documentCoordinator - -echo work method in HttpServerJob -addEvent workHttpServerJob work@HttpServerJob.cpp - -echo work method in RestDocumentHandler -addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp -addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp -addEvent handleRequest handleRequest@HttpServer.cpp -addEvent handleWrite handleWrite@SocketTask.cpp - -addEvent tcp_sendmsg -addEvent tcp_recvmsg - -echo Done. +main "$@" diff --git a/scripts/startLocalCluster.sh b/scripts/startLocalCluster.sh index d0924bc501..fe83e3ed53 100755 --- a/scripts/startLocalCluster.sh +++ b/scripts/startLocalCluster.sh @@ -96,6 +96,7 @@ start() { PORT=$2 mkdir cluster/data$PORT echo Starting $TYPE on port $PORT + mkdir -p cluster/apps$PORT build/bin/arangod -c none \ --database.directory cluster/data$PORT \ --cluster.agency-endpoint tcp://127.0.0.1:$BASE \