mirror of https://gitee.com/bigwinds/arangodb
Merge branch 'devel' of github.com:arangodb/arangodb into devel
This commit is contained in:
Frank Celler
commit
9bb239b737
|
|
@ -48,6 +48,8 @@ devel
|
|||
v3.0.6 (XXXX-XX-XX)
|
||||
-------------------
|
||||
|
||||
* fixed issue #2026
|
||||
|
||||
* slightly better error diagnostics for AQL query compilation and replication
|
||||
|
||||
* fixed issue #2018
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ endif ()
|
|||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
# where to find CMAKE modules
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
|
||||
|
||||
# be verbose about flags used
|
||||
option(VERBOSE OFF)
|
||||
|
|
@ -708,7 +708,7 @@ if (NOT USE_BOOST_UNITTESTS)
|
|||
message(STATUS "BOOST unit-tests are disabled")
|
||||
endif ()
|
||||
|
||||
include_directories(${Boost_INCLUDE_DIR})
|
||||
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
|
||||
add_definitions(-DARANGODB_BOOST_VERSION=\"${Boost_VERSION}\")
|
||||
|
||||
################################################################################
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ For unattended installation, you can set the password using the
|
|||
[debconf helpers](http://www.microhowto.info/howto/perform_an_unattended_installation_of_a_debian_package.html).
|
||||
|
||||
```
|
||||
echo arangodb3 arangodb/password password NEWPASSWORD | debconf-set-selections
|
||||
echo arangodb3 arangodb/password_again password NEWPASSWORD | debconf-set-selections
|
||||
echo arangodb3 arangodb3/password password NEWPASSWORD | debconf-set-selections
|
||||
echo arangodb3 arangodb3/password_again password NEWPASSWORD | debconf-set-selections
|
||||
```
|
||||
|
||||
The commands should be executed prior to the installation.
|
||||
|
|
|
|||
|
|
@ -94,8 +94,19 @@ and the hash value is used to determine the target shard.
|
|||
**Note**: Values of shard key attributes cannot be changed once set.
|
||||
This option is meaningless in a single server setup.
|
||||
|
||||
@RESTBODYPARAM{replicationFactor,integer,optional,int64}
|
||||
(The default is *1*): in a cluster, this attribute determines how many copies
|
||||
of each shard are kept on different DBServers. The value 1 means that only one
|
||||
copy (no synchronous replication) is kept. A value of k means that k-1 replicas
|
||||
are kept. Any two copies reside on different DBServers. Replication between them is
|
||||
synchronous, that is, every write operation to the "leader" copy will be replicated
|
||||
to all "follower" replicas, before the write operation is reported successful.
|
||||
|
||||
If a server fails, this is detected automatically and one of the servers holding
|
||||
copies take over, usually without an error being reported.
|
||||
|
||||
@RESTDESCRIPTION
|
||||
Creates an new collection with a given name. The request must contain an
|
||||
Creates a new collection with a given name. The request must contain an
|
||||
object with the following attributes.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -101,16 +101,6 @@ if test -f last_compiled_version.sha; then
|
|||
fi
|
||||
|
||||
COMPILE_MATTERS="3rdParty"
|
||||
CLEAN_IT=1
|
||||
|
||||
if test -n "$LASTREV"; then
|
||||
lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l`
|
||||
|
||||
if test $lines -eq 0; then
|
||||
echo "no relevant changes, no need for full recompile"
|
||||
CLEAN_IT=0
|
||||
fi
|
||||
fi
|
||||
|
||||
# setup make options
|
||||
if test -z "${CXX}"; then
|
||||
|
|
@ -193,6 +183,8 @@ case "$1" in
|
|||
;;
|
||||
esac
|
||||
|
||||
CLEAN_IT=0
|
||||
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
|
|
@ -303,7 +295,11 @@ while [ $# -gt 0 ]; do
|
|||
TARGET_DIR=$1
|
||||
shift
|
||||
;;
|
||||
|
||||
|
||||
--checkCleanBuild)
|
||||
CLEAN_IT=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
|
|
@ -311,6 +307,18 @@ while [ $# -gt 0 ]; do
|
|||
esac
|
||||
done
|
||||
|
||||
|
||||
if test -n "$LASTREV"; then
|
||||
lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l`
|
||||
|
||||
if test $lines -eq 0; then
|
||||
echo "no relevant changes, no need for full recompile"
|
||||
CLEAN_IT=0
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
|
||||
if [ "$GCC5" == 1 ]; then
|
||||
CC=/usr/bin/gcc-5
|
||||
CXX=/usr/bin/g++-5
|
||||
|
|
|
|||
|
|
@ -946,7 +946,7 @@ Function default_installation_directory
|
|||
Return
|
||||
FunctionEnd
|
||||
|
||||
Function assign_proper_access_rigths
|
||||
Function assign_proper_access_rights
|
||||
StrCpy $0 "0"
|
||||
AccessControl::GrantOnFile \
|
||||
"$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
|
||||
|
|
@ -963,7 +963,7 @@ Function is_writable
|
|||
; is does not matter if we do some errors here
|
||||
${If} $TRI_INSTALL_ALL_USERS == '1'
|
||||
CreateDirectory $INSTDIR
|
||||
Call assign_proper_access_rigths
|
||||
Call assign_proper_access_rights
|
||||
${EndIf}
|
||||
FunctionEnd
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
; CPack install script designed for a nmake build
|
||||
; TODO !addplugindir '@CPACK_PLUGIN_PATH@/AccessControl/Plugins'
|
||||
|
||||
;--------------------------------
|
||||
; Include LogicLib for more readable code
|
||||
|
|
@ -484,14 +485,10 @@ FunctionEnd
|
|||
|
||||
;--------------------------------
|
||||
;Pages
|
||||
!define MUI_PAGE_CUSTOMFUNCTION_PRE skip_page
|
||||
!insertmacro MUI_PAGE_WELCOME
|
||||
|
||||
!define MUI_PAGE_CUSTOMFUNCTION_PRE skip_page
|
||||
!insertmacro MUI_PAGE_LICENSE "@CPACK_RESOURCE_FILE_LICENSE@"
|
||||
|
||||
Page custom InstallOptionsPage skip_page
|
||||
|
||||
!define MUI_PAGE_CUSTOMFUNCTION_PRE default_installation_directory
|
||||
!define MUI_PAGE_CUSTOMFUNCTION_LEAVE check_installation_directory
|
||||
!insertmacro MUI_PAGE_DIRECTORY
|
||||
|
|
@ -709,7 +706,6 @@ displayAgain:
|
|||
StrCmp $PASSWORD $PASSWORD_AGAIN +3 0
|
||||
MessageBox MB_OK|MB_ICONSTOP "Passwords don't match, try again"
|
||||
Goto displayAgain
|
||||
done:
|
||||
Pop ${TEMP1}
|
||||
|
||||
Return
|
||||
|
|
@ -725,20 +721,17 @@ FunctionEnd
|
|||
Function default_installation_directory
|
||||
; Read variables which defines if arango should be installed as Service
|
||||
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R2 "NSIS.InstallOptions.ini" "Field 2" "State"
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R3 "NSIS.InstallOptions.ini" "Field 3" "State"
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R4 "NSIS.InstallOptions.ini" "Field 4" "State"
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R2 "NSIS.InstallOptions.ini" "Field 2" "State"
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R3 "NSIS.InstallOptions.ini" "Field 3" "State"
|
||||
!insertmacro MUI_INSTALLOPTIONS_READ $R4 "NSIS.InstallOptions.ini" "Field 4" "State"
|
||||
|
||||
${If} $R3 == '1'
|
||||
StrCpy $TRI_INSTALL_TYPE 'AllUsers'
|
||||
${EndIf}
|
||||
|
||||
${If} $R4 == '1'
|
||||
StrCpy $TRI_INSTALL_TYPE 'SingleUser'
|
||||
${EndIf}
|
||||
Call read_options
|
||||
${If} $R3 == '1'
|
||||
StrCpy $TRI_INSTALL_TYPE 'AllUsers'
|
||||
${EndIf}
|
||||
|
||||
${If} $R4 == '1'
|
||||
StrCpy $TRI_INSTALL_TYPE 'SingleUser'
|
||||
${EndIf}
|
||||
|
||||
${Switch} $TRI_INSTALL_TYPE
|
||||
${Case} 'SingleUser'
|
||||
|
|
@ -750,25 +743,25 @@ Function default_installation_directory
|
|||
Return
|
||||
FunctionEnd
|
||||
|
||||
Function assign_proper_access_rigths
|
||||
StrCpy $0 "0"
|
||||
AccessControl::GrantOnFile \
|
||||
"$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
|
||||
Pop $R0
|
||||
${If} $R0 == error
|
||||
Pop $R0
|
||||
StrCpy $0 "1"
|
||||
DetailPrint `AccessControl error: $R0`
|
||||
; MessageBox MB_OK "target directory $INSTDIR can not get cannot get correct access rigths"
|
||||
${EndIf}
|
||||
FunctionEnd
|
||||
; TODO Function assign_proper_access_rights
|
||||
; TODO StrCpy $0 "0"
|
||||
; TODO AccessControl::GrantOnFile \
|
||||
; TODO "$INSTDIR" "(BU)" "GenericRead + GenericWrite + GenericExecute"
|
||||
; TODO Pop $R0
|
||||
; TODO ${If} $R0 == error
|
||||
; TODO Pop $R0
|
||||
; TODO StrCpy $0 "1"
|
||||
; TODO DetailPrint `AccessControl error: $R0`
|
||||
; TODO ; MessageBox MB_OK "target directory $INSTDIR can not get cannot get correct access rigths"
|
||||
; TODO ${EndIf}
|
||||
; TODO FunctionEnd
|
||||
|
||||
Function is_writable
|
||||
; is does not matter if we do some errors here
|
||||
${If} $TRI_INSTALL_ALL_USERS == '1'
|
||||
CreateDirectory $INSTDIR
|
||||
Call assign_proper_access_rigths
|
||||
${EndIf}
|
||||
; TODO Call assign_proper_access_rights
|
||||
${EndIf}
|
||||
FunctionEnd
|
||||
|
||||
Function check_installation_directory
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ if [ "$1" = "configure" -a -z "$2" ]; then
|
|||
/usr/sbin/arango-init-database \
|
||||
--uid arangodb --gid arangodb || true
|
||||
fi
|
||||
|
||||
db_set arangodb3/password_again ""
|
||||
db_set arangodb3/password ""
|
||||
db_go
|
||||
fi
|
||||
|
||||
# check if we should upgrade the database directory
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
set -e
|
||||
|
||||
getent group arangodb >/dev/null || groupadd -r arangodb
|
||||
getent passwd arangodb >/dev/null || useradd -r -g arangodb -d /usr/share/arangodb -s /bin/false -c "ArangoDB Application User" arangodb
|
||||
getent passwd arangodb >/dev/null || useradd -r -g arangodb -d /usr/share/arangodb3 -s /bin/false -c "ArangoDB Application User" arangodb
|
||||
|
||||
install -o arangodb -g arangodb -m 755 -d /var/lib/arangodb3
|
||||
install -o arangodb -g arangodb -m 755 -d /var/lib/arangodb3-apps
|
||||
|
|
|
|||
|
|
@ -50,6 +50,11 @@ fi
|
|||
|
||||
VERSION="$1"
|
||||
|
||||
if echo ${VERSION} | grep -q -- '-'; then
|
||||
echo "${VERSION} mustn't contain minuses! "
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if git tag | grep -q "^v$VERSION$"; then
|
||||
echo "$0: version $VERSION already defined"
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -476,7 +476,7 @@ bool config_t::merge(VPackSlice const& conf) {
|
|||
ss << "Min RAFT interval: ";
|
||||
if (_minPing == 0) { // Command line beats persistence
|
||||
if (conf.hasKey(minPingStr)) {
|
||||
_minPing = conf.get(minPingStr).getUInt();
|
||||
_minPing = conf.get(minPingStr).getDouble();
|
||||
ss << _minPing << " (persisted)";
|
||||
} else {
|
||||
_minPing = 0.5;
|
||||
|
|
@ -491,7 +491,7 @@ bool config_t::merge(VPackSlice const& conf) {
|
|||
ss << "Max RAFT interval: ";
|
||||
if (_maxPing == 0) { // Command line beats persistence
|
||||
if (conf.hasKey(maxPingStr)) {
|
||||
_maxPing = conf.get(maxPingStr).getUInt();
|
||||
_maxPing = conf.get(maxPingStr).getDouble();
|
||||
ss << _maxPing << " (persisted)";
|
||||
} else {
|
||||
_maxPing = 2.5;
|
||||
|
|
|
|||
|
|
@ -192,6 +192,7 @@ struct UserVarFinder final : public WalkerWorker<ExecutionNode> {
|
|||
en->getType() == ExecutionNode::INDEX ||
|
||||
en->getType() == ExecutionNode::ENUMERATE_LIST ||
|
||||
en->getType() == ExecutionNode::TRAVERSAL ||
|
||||
en->getType() == ExecutionNode::SHORTEST_PATH ||
|
||||
en->getType() == ExecutionNode::COLLECT) {
|
||||
depth += 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -581,7 +581,7 @@ ExecutionNode const* ExecutionNode::getLoop() const {
|
|||
auto type = node->getType();
|
||||
|
||||
if (type == ENUMERATE_COLLECTION || type == INDEX || type == TRAVERSAL ||
|
||||
type == ENUMERATE_LIST) {
|
||||
type == ENUMERATE_LIST || type == SHORTEST_PATH) {
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
|
@ -1147,7 +1147,7 @@ void ExecutionNode::RegisterPlan::after(ExecutionNode* en) {
|
|||
en->getVarsUsedLater();
|
||||
std::vector<Variable const*> const& varsUsedHere =
|
||||
en->getVariablesUsedHere();
|
||||
|
||||
|
||||
// We need to delete those variables that have been used here but are not
|
||||
// used any more later:
|
||||
std::unordered_set<RegisterId> regsToClear;
|
||||
|
|
@ -1160,7 +1160,7 @@ void ExecutionNode::RegisterPlan::after(ExecutionNode* en) {
|
|||
|
||||
if (it2 == varInfo.end()) {
|
||||
// report an error here to prevent crashing
|
||||
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "missing variable #" + std::to_string(v->id) + " for node " + en->getTypeString() + " while planning registers");
|
||||
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "missing variable #" + std::to_string(v->id) + " (" + v->name + ") for node " + en->getTypeString() + " while planning registers");
|
||||
}
|
||||
|
||||
// finally adjust the variable inside the IN calculation
|
||||
|
|
|
|||
|
|
@ -749,7 +749,7 @@ ExecutionNode* ExecutionPlan::fromNodeTraversal(ExecutionNode* previous,
|
|||
return addDependency(previous, en);
|
||||
}
|
||||
|
||||
AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode* previous,
|
||||
AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode*& previous,
|
||||
AstNode const* vertex) {
|
||||
if (vertex->type == NODE_TYPE_OBJECT && vertex->isConstant()) {
|
||||
size_t n = vertex->numMembers();
|
||||
|
|
@ -767,6 +767,8 @@ AstNode const* ExecutionPlan::parseTraversalVertexNode(ExecutionNode* previous,
|
|||
// operand is some misc expression
|
||||
auto calc = createTemporaryCalculation(vertex, previous);
|
||||
vertex = _ast->createNodeReference(getOutVariable(calc));
|
||||
// update previous so the caller has an updated value
|
||||
previous = calc;
|
||||
}
|
||||
|
||||
return vertex;
|
||||
|
|
@ -1960,6 +1962,7 @@ bool ExecutionPlan::isDeadSimple() const {
|
|||
nodeType == ExecutionNode::ENUMERATE_COLLECTION ||
|
||||
nodeType == ExecutionNode::ENUMERATE_LIST ||
|
||||
nodeType == ExecutionNode::TRAVERSAL ||
|
||||
nodeType == ExecutionNode::SHORTEST_PATH ||
|
||||
nodeType == ExecutionNode::INDEX) {
|
||||
// these node types are not simple
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -286,7 +286,7 @@ class ExecutionPlan {
|
|||
ExecutionNode* fromJson(arangodb::basics::Json const& Json);
|
||||
|
||||
/// @brief create an vertex element for graph nodes
|
||||
AstNode const* parseTraversalVertexNode(ExecutionNode*, AstNode const*);
|
||||
AstNode const* parseTraversalVertexNode(ExecutionNode*&, AstNode const*);
|
||||
|
||||
private:
|
||||
/// @brief map from node id to the actual node
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
#include "Aql/Function.h"
|
||||
#include "Aql/IndexNode.h"
|
||||
#include "Aql/ModificationNodes.h"
|
||||
#include "Aql/ShortestPathNode.h"
|
||||
#include "Aql/SortCondition.h"
|
||||
#include "Aql/SortNode.h"
|
||||
#include "Aql/TraversalConditionFinder.h"
|
||||
|
|
@ -324,7 +325,8 @@ void arangodb::aql::removeRedundantSortsRule(Optimizer* opt,
|
|||
}
|
||||
} else if (current->getType() == EN::ENUMERATE_LIST ||
|
||||
current->getType() == EN::ENUMERATE_COLLECTION ||
|
||||
current->getType() == EN::TRAVERSAL) {
|
||||
current->getType() == EN::TRAVERSAL ||
|
||||
current->getType() == EN::SHORTEST_PATH) {
|
||||
// ok, but we cannot remove two different sorts if one of these node
|
||||
// types is between them
|
||||
// example: in the following query, the one sort will be optimized
|
||||
|
|
@ -764,10 +766,10 @@ void arangodb::aql::removeSortRandRule(Optimizer* opt, ExecutionPlan* plan,
|
|||
case EN::SUBQUERY:
|
||||
case EN::ENUMERATE_LIST:
|
||||
case EN::TRAVERSAL:
|
||||
case EN::SHORTEST_PATH:
|
||||
case EN::INDEX: {
|
||||
// if we found another SortNode, an CollectNode, FilterNode, a
|
||||
// SubqueryNode,
|
||||
// an EnumerateListNode, a TraversalNode or an IndexNode
|
||||
// if we found another SortNode, a CollectNode, FilterNode, a
|
||||
// SubqueryNode, an EnumerateListNode, a TraversalNode or an IndexNode
|
||||
// this means we cannot apply our optimization
|
||||
collectionNode = nullptr;
|
||||
current = nullptr;
|
||||
|
|
@ -949,7 +951,9 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
|
|||
} else if (currentType == EN::INDEX ||
|
||||
currentType == EN::ENUMERATE_COLLECTION ||
|
||||
currentType == EN::ENUMERATE_LIST ||
|
||||
currentType == EN::TRAVERSAL || currentType == EN::COLLECT ||
|
||||
currentType == EN::TRAVERSAL ||
|
||||
currentType == EN::SHORTEST_PATH ||
|
||||
currentType == EN::COLLECT ||
|
||||
currentType == EN::NORESULTS) {
|
||||
// we will not push further down than such nodes
|
||||
shouldMove = false;
|
||||
|
|
@ -1241,6 +1245,17 @@ class arangodb::aql::RedundantCalculationsReplacer final
|
|||
std::unordered_map<VariableId, Variable const*> const& replacements)
|
||||
: _replacements(replacements) {
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void replaceStartTargetVariables(ExecutionNode* en) {
|
||||
auto node = static_cast<T*>(en);
|
||||
if (node->_inStartVariable != nullptr) {
|
||||
node->_inStartVariable = Variable::replace(node->_inStartVariable, _replacements);
|
||||
}
|
||||
if (node->_inTargetVariable != nullptr) {
|
||||
node->_inTargetVariable = Variable::replace(node->_inTargetVariable, _replacements);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void replaceInVariable(ExecutionNode* en) {
|
||||
|
|
@ -1290,6 +1305,11 @@ class arangodb::aql::RedundantCalculationsReplacer final
|
|||
replaceInVariable<TraversalNode>(en);
|
||||
break;
|
||||
}
|
||||
|
||||
case EN::SHORTEST_PATH: {
|
||||
replaceStartTargetVariables<ShortestPathNode>(en);
|
||||
break;
|
||||
}
|
||||
|
||||
case EN::COLLECT: {
|
||||
auto node = static_cast<CollectNode*>(en);
|
||||
|
|
@ -3589,7 +3609,7 @@ void arangodb::aql::patchUpdateStatementsRule(Optimizer* opt,
|
|||
}
|
||||
}
|
||||
|
||||
if (type == EN::TRAVERSAL) {
|
||||
if (type == EN::TRAVERSAL || type == EN::SHORTEST_PATH) {
|
||||
// unclear what will be read by the traversal
|
||||
modified = false;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ namespace aql {
|
|||
/// @brief class ShortestPathNode
|
||||
class ShortestPathNode : public ExecutionNode {
|
||||
friend class ExecutionBlock;
|
||||
friend class RedundantCalculationsReplacer;
|
||||
friend class ShortestPathBlock;
|
||||
|
||||
/// @brief constructor with a vocbase and a collection name
|
||||
|
|
|
|||
|
|
@ -123,6 +123,11 @@ static void mergeResults(
|
|||
resultBody->openArray();
|
||||
for (auto const& pair : reverseMapping) {
|
||||
VPackSlice arr = resultMap.find(pair.first)->second->slice();
|
||||
if (arr.isObject() && arr.hasKey("error") && arr.get("error").isBoolean() && arr.get("error").getBoolean()) {
|
||||
// an error occurred, now rethrow the error
|
||||
int res = arr.get("errorNum").getNumericValue<int>();
|
||||
THROW_ARANGO_EXCEPTION(res);
|
||||
}
|
||||
resultBody->add(arr.at(pair.second));
|
||||
}
|
||||
resultBody->close();
|
||||
|
|
@ -736,6 +741,7 @@ int createDocumentOnCoordinator(
|
|||
bool useMultiple = slice.isArray();
|
||||
|
||||
int res = TRI_ERROR_NO_ERROR;
|
||||
|
||||
if (useMultiple) {
|
||||
VPackValueLength length = slice.length();
|
||||
for (VPackValueLength idx = 0; idx < length; ++idx) {
|
||||
|
|
@ -766,6 +772,7 @@ int createDocumentOnCoordinator(
|
|||
// Now prepare the requests:
|
||||
std::vector<ClusterCommRequest> requests;
|
||||
auto body = std::make_shared<std::string>();
|
||||
|
||||
for (auto const& it : shardMap) {
|
||||
if (!useMultiple) {
|
||||
TRI_ASSERT(it.second.size() == 1);
|
||||
|
|
@ -801,7 +808,7 @@ int createDocumentOnCoordinator(
|
|||
"shard:" + it.first, arangodb::GeneralRequest::RequestType::POST,
|
||||
baseUrl + StringUtils::urlEncode(it.first) + optsUrlPart, body);
|
||||
}
|
||||
|
||||
|
||||
// Perform the requests
|
||||
size_t nrDone = 0;
|
||||
cc->performRequests(requests, CL_DEFAULT_TIMEOUT, nrDone, Logger::REQUESTS);
|
||||
|
|
|
|||
|
|
@ -47,9 +47,9 @@ class HttpResponse;
|
|||
namespace rest {
|
||||
class GeneralServer;
|
||||
|
||||
size_t const HttpCommTask::MaximalHeaderSize = 1 * 1024 * 1024; // 1 MB
|
||||
size_t const HttpCommTask::MaximalBodySize = 512 * 1024 * 1024; // 512 MB
|
||||
size_t const HttpCommTask::MaximalPipelineSize = 512 * 1024 * 1024; // 512 MB
|
||||
size_t const HttpCommTask::MaximalHeaderSize = 2 * 1024 * 1024; // 2 MB
|
||||
size_t const HttpCommTask::MaximalBodySize = 1024 * 1024 * 1024; // 1024 MB
|
||||
size_t const HttpCommTask::MaximalPipelineSize = 1024 * 1024 * 1024; // 1024 MB
|
||||
size_t const HttpCommTask::RunCompactEvery = 500;
|
||||
|
||||
HttpCommTask::HttpCommTask(GeneralServer* server, TRI_socket_t sock,
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ void BootstrapFeature::start() {
|
|||
auto vocbase = DatabaseFeature::DATABASE->vocbase();
|
||||
|
||||
auto ss = ServerState::instance();
|
||||
if (!ss->isRunningInCluster() && !ss->isAgent()) {
|
||||
if (!ss->isRunningInCluster()) {
|
||||
LOG_TOPIC(DEBUG, Logger::STARTUP) << "Running server/server.js";
|
||||
V8DealerFeature::DEALER->loadJavascript(vocbase, "server/server.js");
|
||||
} else if (ss->isCoordinator()) {
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@
|
|||
#include "VocBase/server.h"
|
||||
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
|
||||
// #define DEBUG_DATAFILE 1
|
||||
|
||||
|
|
@ -756,7 +757,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
|
|||
bool nextMarkerOk = false;
|
||||
|
||||
if (size > 0) {
|
||||
auto next = reinterpret_cast<char const*>(marker) + size;
|
||||
auto next = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
|
||||
auto p = next;
|
||||
|
||||
if (p < end) {
|
||||
|
|
@ -783,7 +784,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
|
|||
// there is a next marker
|
||||
auto nextMarker =
|
||||
reinterpret_cast<TRI_df_marker_t const*>(next);
|
||||
|
||||
|
||||
if (nextMarker->getType() != 0 &&
|
||||
nextMarker->getSize() >= sizeof(TRI_df_marker_t) &&
|
||||
next + nextMarker->getSize() <= end &&
|
||||
|
|
@ -810,10 +811,67 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
|
|||
datafile->_next = datafile->_data + datafile->_currentSize;
|
||||
datafile->_state = TRI_DF_STATE_OPEN_ERROR;
|
||||
|
||||
LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' at position " << currentSize << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
|
||||
LOG(WARN) << "crc mismatch found in datafile '" << datafile->getName(datafile) << "' of size "
|
||||
<< datafile->_maximalSize << ", at position " << currentSize;
|
||||
|
||||
LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker)
|
||||
<< "' and size " << size
|
||||
<< ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc();
|
||||
|
||||
{
|
||||
LOG(INFO) << "raw marker data following:";
|
||||
char const* p = reinterpret_cast<char const*>(marker);
|
||||
char const* e = reinterpret_cast<char const*>(marker) + DatafileHelper::AlignedSize<size_t>(size);
|
||||
std::string line;
|
||||
std::string raw;
|
||||
size_t printed = 0;
|
||||
while (p < e) {
|
||||
// print offset
|
||||
line.append("0x");
|
||||
uintptr_t offset = static_cast<uintptr_t>(p - datafile->_data);
|
||||
for (size_t i = 0; i < 8; ++i) {
|
||||
uint8_t c = static_cast<uint8_t>((offset & (0xFFULL << 8 * (7 - i))) >> 8 * (7 - i));
|
||||
uint8_t n1 = c >> 4;
|
||||
uint8_t n2 = c & 0x0F;
|
||||
|
||||
line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
|
||||
line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
|
||||
}
|
||||
|
||||
// print data
|
||||
line.append(": ");
|
||||
for (size_t i = 0; i < 16; ++i) {
|
||||
if (p >= e) {
|
||||
line.append(" ");
|
||||
} else {
|
||||
uint8_t c = static_cast<uint8_t>(*p++);
|
||||
uint8_t n1 = c >> 4;
|
||||
uint8_t n2 = c & 0x0F;
|
||||
|
||||
line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10);
|
||||
line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10);
|
||||
line.push_back(' ');
|
||||
|
||||
raw.push_back((c < 32 || c >= 127) ? '.' : static_cast<unsigned char>(c));
|
||||
|
||||
++printed;
|
||||
}
|
||||
}
|
||||
|
||||
LOG(INFO) << line << " " << raw;
|
||||
line.clear();
|
||||
raw.clear();
|
||||
|
||||
if (printed >= 2048) {
|
||||
LOG(INFO) << "(output truncated due to excessive length)";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (nextMarkerOk) {
|
||||
LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it";
|
||||
LOG(INFO) << "please restart the server with the parameter '--wal.ignore-logfile-errors true' to repair the marker";
|
||||
} else {
|
||||
LOG(WARN) << "data directly following this marker cannot be analyzed";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -161,6 +161,16 @@ void ImportFeature::validateOptions(
|
|||
StringUtils::join(positionals, ", ");
|
||||
FATAL_ERROR_EXIT();
|
||||
}
|
||||
|
||||
static unsigned const MaxBatchSize = 768 * 1024 * 1024;
|
||||
|
||||
if (_chunkSize > MaxBatchSize) {
|
||||
// it's not sensible to raise the batch size beyond this value
|
||||
// because the server has a built-in limit for the batch size too
|
||||
// and will reject bigger HTTP request bodies
|
||||
LOG(WARN) << "capping --batch-size value to " << MaxBatchSize;
|
||||
_chunkSize = MaxBatchSize;
|
||||
}
|
||||
}
|
||||
|
||||
void ImportFeature::start() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
#.rst:
|
||||
# FindPythonInterp
|
||||
# ----------------
|
||||
#
|
||||
# Find python interpreter
|
||||
#
|
||||
# This module finds if Python interpreter is installed and determines
|
||||
# where the executables are. This code sets the following variables:
|
||||
#
|
||||
# ::
|
||||
#
|
||||
# PYTHONINTERP_FOUND - Was the Python executable found
|
||||
# PYTHON_EXECUTABLE - path to the Python interpreter
|
||||
#
|
||||
#
|
||||
#
|
||||
# ::
|
||||
#
|
||||
# PYTHON_VERSION_STRING - Python version found e.g. 2.5.2
|
||||
# PYTHON_VERSION_MAJOR - Python major version found e.g. 2
|
||||
# PYTHON_VERSION_MINOR - Python minor version found e.g. 5
|
||||
# PYTHON_VERSION_PATCH - Python patch version found e.g. 2
|
||||
#
|
||||
#
|
||||
#
|
||||
# The Python_ADDITIONAL_VERSIONS variable can be used to specify a list
|
||||
# of version numbers that should be taken into account when searching
|
||||
# for Python. You need to set this variable before calling
|
||||
# find_package(PythonInterp).
|
||||
#
|
||||
# If calling both ``find_package(PythonInterp)`` and
|
||||
# ``find_package(PythonLibs)``, call ``find_package(PythonInterp)`` first to
|
||||
# get the currently active Python version by default with a consistent version
|
||||
# of PYTHON_LIBRARIES.
|
||||
|
||||
#=============================================================================
|
||||
# Copyright 2005-2010 Kitware, Inc.
|
||||
# Copyright 2011 Bjoern Ricks <bjoern.ricks@gmail.com>
|
||||
# Copyright 2012 Rolf Eike Beer <eike@sf-mail.de>
|
||||
#
|
||||
# Distributed under the OSI-approved BSD License (the "License");
|
||||
# see accompanying file Copyright.txt for details.
|
||||
#
|
||||
# This software is distributed WITHOUT ANY WARRANTY; without even the
|
||||
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See the License for more information.
|
||||
#=============================================================================
|
||||
# (To distribute this file outside of CMake, substitute the full
|
||||
# License text for the above reference.)
|
||||
|
||||
unset(_Python_NAMES)
|
||||
|
||||
set(_PYTHON1_VERSIONS 1.6 1.5)
|
||||
set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0)
|
||||
set(_PYTHON3_VERSIONS 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
|
||||
|
||||
if(PythonInterp_FIND_VERSION)
|
||||
if(PythonInterp_FIND_VERSION_COUNT GREATER 1)
|
||||
set(_PYTHON_FIND_MAJ_MIN "${PythonInterp_FIND_VERSION_MAJOR}.${PythonInterp_FIND_VERSION_MINOR}")
|
||||
list(APPEND _Python_NAMES
|
||||
python${_PYTHON_FIND_MAJ_MIN}
|
||||
python${PythonInterp_FIND_VERSION_MAJOR})
|
||||
unset(_PYTHON_FIND_OTHER_VERSIONS)
|
||||
if(NOT PythonInterp_FIND_VERSION_EXACT)
|
||||
foreach(_PYTHON_V ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS})
|
||||
if(NOT _PYTHON_V VERSION_LESS _PYTHON_FIND_MAJ_MIN)
|
||||
list(APPEND _PYTHON_FIND_OTHER_VERSIONS ${_PYTHON_V})
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
unset(_PYTHON_FIND_MAJ_MIN)
|
||||
else()
|
||||
list(APPEND _Python_NAMES python${PythonInterp_FIND_VERSION_MAJOR})
|
||||
set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON${PythonInterp_FIND_VERSION_MAJOR}_VERSIONS})
|
||||
endif()
|
||||
else()
|
||||
set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON3_VERSIONS} ${_PYTHON2_VERSIONS} ${_PYTHON1_VERSIONS})
|
||||
endif()
|
||||
find_program(PYTHON_EXECUTABLE NAMES ${_Python_NAMES})
|
||||
|
||||
# Set up the versions we know about, in the order we will search. Always add
|
||||
# the user supplied additional versions to the front.
|
||||
set(_Python_VERSIONS ${Python_ADDITIONAL_VERSIONS})
|
||||
# If FindPythonInterp has already found the major and minor version,
|
||||
# insert that version next to get consistent versions of the interpreter and
|
||||
# library.
|
||||
if(DEFINED PYTHONLIBS_VERSION_STRING)
|
||||
string(REPLACE "." ";" _PYTHONLIBS_VERSION "${PYTHONLIBS_VERSION_STRING}")
|
||||
list(GET _PYTHONLIBS_VERSION 0 _PYTHONLIBS_VERSION_MAJOR)
|
||||
list(GET _PYTHONLIBS_VERSION 1 _PYTHONLIBS_VERSION_MINOR)
|
||||
list(APPEND _Python_VERSIONS ${_PYTHONLIBS_VERSION_MAJOR}.${_PYTHONLIBS_VERSION_MINOR})
|
||||
endif()
|
||||
# Search for the current active python version first
|
||||
list(APPEND _Python_VERSIONS ";")
|
||||
list(APPEND _Python_VERSIONS ${_PYTHON_FIND_OTHER_VERSIONS})
|
||||
|
||||
unset(_PYTHON_FIND_OTHER_VERSIONS)
|
||||
unset(_PYTHON1_VERSIONS)
|
||||
unset(_PYTHON2_VERSIONS)
|
||||
unset(_PYTHON3_VERSIONS)
|
||||
|
||||
# Search for newest python version if python executable isn't found
|
||||
if(NOT PYTHON_EXECUTABLE)
|
||||
foreach(_CURRENT_VERSION IN LISTS _Python_VERSIONS)
|
||||
set(_Python_NAMES python${_CURRENT_VERSION})
|
||||
if(WIN32)
|
||||
list(APPEND _Python_NAMES python)
|
||||
endif()
|
||||
find_program(PYTHON_EXECUTABLE
|
||||
NAMES ${_Python_NAMES}
|
||||
PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]
|
||||
)
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# determine python version string
|
||||
if(PYTHON_EXECUTABLE)
|
||||
execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c
|
||||
"import sys; sys.stdout.write(';'.join([str(x) for x in sys.version_info[:3]]))"
|
||||
OUTPUT_VARIABLE _VERSION
|
||||
RESULT_VARIABLE _PYTHON_VERSION_RESULT
|
||||
ERROR_QUIET)
|
||||
if(NOT _PYTHON_VERSION_RESULT)
|
||||
string(REPLACE ";" "." PYTHON_VERSION_STRING "${_VERSION}")
|
||||
list(GET _VERSION 0 PYTHON_VERSION_MAJOR)
|
||||
list(GET _VERSION 1 PYTHON_VERSION_MINOR)
|
||||
list(GET _VERSION 2 PYTHON_VERSION_PATCH)
|
||||
if(PYTHON_VERSION_PATCH EQUAL 0)
|
||||
# it's called "Python 2.7", not "2.7.0"
|
||||
string(REGEX REPLACE "\\.0$" "" PYTHON_VERSION_STRING "${PYTHON_VERSION_STRING}")
|
||||
endif()
|
||||
else()
|
||||
# sys.version predates sys.version_info, so use that
|
||||
execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c "import sys; sys.stdout.write(sys.version)"
|
||||
OUTPUT_VARIABLE _VERSION
|
||||
RESULT_VARIABLE _PYTHON_VERSION_RESULT
|
||||
ERROR_QUIET)
|
||||
if(NOT _PYTHON_VERSION_RESULT)
|
||||
string(REGEX REPLACE " .*" "" PYTHON_VERSION_STRING "${_VERSION}")
|
||||
string(REGEX REPLACE "^([0-9]+)\\.[0-9]+.*" "\\1" PYTHON_VERSION_MAJOR "${PYTHON_VERSION_STRING}")
|
||||
string(REGEX REPLACE "^[0-9]+\\.([0-9])+.*" "\\1" PYTHON_VERSION_MINOR "${PYTHON_VERSION_STRING}")
|
||||
if(PYTHON_VERSION_STRING MATCHES "^[0-9]+\\.[0-9]+\\.([0-9]+)")
|
||||
set(PYTHON_VERSION_PATCH "${CMAKE_MATCH_1}")
|
||||
else()
|
||||
set(PYTHON_VERSION_PATCH "0")
|
||||
endif()
|
||||
else()
|
||||
# sys.version was first documented for Python 1.5, so assume
|
||||
# this is older.
|
||||
set(PYTHON_VERSION_STRING "1.4")
|
||||
set(PYTHON_VERSION_MAJOR "1")
|
||||
set(PYTHON_VERSION_MINOR "4")
|
||||
set(PYTHON_VERSION_PATCH "0")
|
||||
endif()
|
||||
endif()
|
||||
unset(_PYTHON_VERSION_RESULT)
|
||||
unset(_VERSION)
|
||||
endif()
|
||||
|
||||
# handle the QUIETLY and REQUIRED arguments and set PYTHONINTERP_FOUND to TRUE if
|
||||
# all listed variables are TRUE
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonInterp REQUIRED_VARS PYTHON_EXECUTABLE VERSION_VAR PYTHON_VERSION_STRING)
|
||||
|
||||
mark_as_advanced(PYTHON_EXECUTABLE)
|
||||
|
|
@ -80,11 +80,12 @@ if(CMAKE_TARGET_ARCHITECTURES MATCHES ".*x86_64.*")
|
|||
elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "aarch64")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "arm64")
|
||||
elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "armv7")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "arm7")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "armhf")
|
||||
else()
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "i386")
|
||||
endif()
|
||||
|
||||
set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE ${ARANGODB_PACKAGE_ARCHITECTURE})
|
||||
set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${ARANGODB_PACKAGE_REVISION}_${ARANGODB_PACKAGE_ARCHITECTURE}")
|
||||
|
||||
################################################################################
|
||||
|
|
@ -98,3 +99,12 @@ include(${ORIGINAL_SOURCE_DIR}/cmake/InstallArangoDBJSClient.cmake)
|
|||
include(${ORIGINAL_SOURCE_DIR}/arangosh/install.cmake)
|
||||
|
||||
include(CPack)
|
||||
|
||||
# TODO: missing usr/share/man
|
||||
# TODO: missing usr/share/man/man1
|
||||
# TODO: missing usr/share/man/man1/arangobench.1
|
||||
# TODO: missing usr/share/man/man1/arangodump.1
|
||||
# TODO: missing usr/share/man/man1/arangoimp.1
|
||||
# TODO: missing usr/share/man/man1/arangorestore.1
|
||||
# TODO: missing usr/share/man/man1/arangosh.1
|
||||
# usr/share/man/man8/foxx-manager.8
|
||||
|
|
|
|||
|
|
@ -24,10 +24,11 @@ if(CMAKE_TARGET_ARCHITECTURES MATCHES ".*x86_64.*")
|
|||
elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "aarch64")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "arm64")
|
||||
elseif(CMAKE_TARGET_ARCHITECTURES MATCHES "armv7")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "arm7")
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "armhf")
|
||||
else()
|
||||
set(ARANGODB_PACKAGE_ARCHITECTURE "i386")
|
||||
endif()
|
||||
set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE ${ARANGODB_PACKAGE_ARCHITECTURE})
|
||||
set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${ARANGODB_PACKAGE_REVISION}_${ARANGODB_PACKAGE_ARCHITECTURE}")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ exports.plainServerVersion = function () {
|
|||
devel = version.match(/(.*)((alpha|beta|devel|rc)[0-9]*)$/);
|
||||
|
||||
if (devel !== null) {
|
||||
version = devel[1];
|
||||
version = devel[1] + '0';
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
// //////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
const isCluster = require("@arangodb/cluster").isCluster();
|
||||
const isAgent = global.ArangoAgent.enabled();
|
||||
|
||||
var _ = require('lodash');
|
||||
var flatten = require('internal').flatten;
|
||||
|
|
@ -231,7 +232,7 @@ function asNumber (num) {
|
|||
}
|
||||
|
||||
function updateQueueDelayClusterAware() {
|
||||
if (isCluster) {
|
||||
if (isCluster && !isAgent) {
|
||||
global.ArangoAgency.set('Current/FoxxmasterQueueupdate', true);
|
||||
}
|
||||
updateQueueDelay();
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ exports.plainServerVersion = function () {
|
|||
devel = version.match(/(.*)((alpha|beta|devel|rc)[0-9]*)$/);
|
||||
|
||||
if (devel !== null) {
|
||||
version = devel[1];
|
||||
version = devel[1] + '0';;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ void SimpleHttpResult::addHeaderField(char const* key, size_t keyLength,
|
|||
|
||||
if (_returnCode == 204) {
|
||||
// HTTP 204 = No content. Assume we will have a content-length of 0.
|
||||
// note that will value can be overridden later if the response has the content-length
|
||||
// note that the value can be overridden later if the response has the content-length
|
||||
// header set to some other value
|
||||
setContentLength(0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,5 +14,6 @@ cd ${DIR}/..
|
|||
--buildDir build-deb \
|
||||
--targetDir /var/tmp/ \
|
||||
--jemalloc \
|
||||
$@
|
||||
|
||||
cd ${DIR}/..
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ cd ${DIR}/..
|
|||
--msvc \
|
||||
--buildDir /cygdrive/c/b/y/ \
|
||||
--package NSIS \
|
||||
--targetDir /var/tmp/
|
||||
--targetDir /var/tmp/ \
|
||||
$@
|
||||
|
||||
cd ${DIR}/..
|
||||
|
|
|
|||
|
|
@ -13,5 +13,6 @@ cd ${DIR}/..
|
|||
--buildDir build-rpm \
|
||||
--targetDir /var/tmp/ \
|
||||
--jemalloc \
|
||||
$@
|
||||
|
||||
cd ${DIR}/..
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
// Compile with
|
||||
// g++ perfanalysis.cpp -o perfanalyis -std=c++11 -Wall -O3
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <unordered_map>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
using namespace std;
|
||||
|
||||
struct Event {
|
||||
static const regex re;
|
||||
string threadName;
|
||||
int tid;
|
||||
string cpu;
|
||||
|
|
@ -22,45 +25,27 @@ struct Event {
|
|||
bool isRet;
|
||||
|
||||
Event(string& line) : isRet(false) {
|
||||
char* s = strdup(line.c_str());
|
||||
char* p = strtok(s, " ");
|
||||
char* q;
|
||||
if (p != nullptr) {
|
||||
threadName = p;
|
||||
p = strtok(nullptr, " ");
|
||||
tid = strtol(p, nullptr, 10);
|
||||
p = strtok(nullptr, " ");
|
||||
cpu = p;
|
||||
p = strtok(nullptr, " ");
|
||||
startTime = strtod(p, nullptr);
|
||||
p = strtok(nullptr, " ");
|
||||
q = strtok(nullptr, " ");
|
||||
if (strcmp(q, "cs:") == 0) {
|
||||
free(s);
|
||||
return;
|
||||
}
|
||||
name = p;
|
||||
name.pop_back();
|
||||
auto l = name.size();
|
||||
if (l >= 3 && name[l-1] == 't' && name[l-2] == 'e' &&
|
||||
name[l-3] == 'R') {
|
||||
isRet = true;
|
||||
name.pop_back();
|
||||
name.pop_back();
|
||||
name.pop_back();
|
||||
}
|
||||
inbrackets = q;
|
||||
std::smatch match_obj;
|
||||
if(!std::regex_search(line, match_obj, re)){
|
||||
throw std::logic_error("could not parse line");
|
||||
}
|
||||
|
||||
threadName = match_obj[1];
|
||||
tid = std::stoi(match_obj[2]);
|
||||
cpu = match_obj[3];
|
||||
startTime = std::stod(match_obj[4]);
|
||||
duration = 0;
|
||||
name = match_obj[6];
|
||||
inbrackets = match_obj[7];
|
||||
if (match_obj[9].length() > 0) {
|
||||
isRet = true;
|
||||
name.erase(name.end() - 3, name.end()); // remove Ret suffix form name
|
||||
}
|
||||
free(s);
|
||||
}
|
||||
|
||||
bool empty() {
|
||||
return name.empty();
|
||||
}
|
||||
bool empty() { return name.empty(); }
|
||||
|
||||
string id() {
|
||||
return to_string(tid) + name;
|
||||
}
|
||||
string id() { return to_string(tid) + name; }
|
||||
|
||||
string pretty() {
|
||||
return to_string(duration) + " " + name + " " + to_string(startTime);
|
||||
|
|
@ -77,31 +62,46 @@ struct Event {
|
|||
}
|
||||
};
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
unordered_map<string, Event*> table;
|
||||
vector<Event*> list;
|
||||
// sample output:
|
||||
// arangod 32636 [005] 16678249.324973: probe_arangod:insertLocalRet: (14a7f60 <- 14a78d6)
|
||||
// process tid core timepoint scope:name frame
|
||||
const regex Event::re(
|
||||
R"_(\s*(\S+))_" // name 1
|
||||
R"_(\s+(\d+))_" // tid 2
|
||||
R"_(\s+\[(\d+)\])_" // cup 3
|
||||
R"_(\s+(\d+\.\d+):)_" // time 4
|
||||
R"_(\s+([^: ]+):([^: ]+):)_" // scope:func 5:6
|
||||
R"_(\s+\(([0-9a-f]+)(\s+<-\s+([0-9a-f]+))?\))_" // (start -> stop) 7 -> 9
|
||||
,
|
||||
std::regex_constants::ECMAScript | std::regex_constants::optimize);
|
||||
|
||||
int main(int /*argc*/, char* /*argv*/ []) {
|
||||
unordered_map<string, unique_ptr<Event>> table;
|
||||
vector<unique_ptr<Event>> list;
|
||||
|
||||
string line;
|
||||
while (getline(cin, line)) {
|
||||
Event* e = new Event(line);
|
||||
if (!e->empty()) {
|
||||
string id = e->id();
|
||||
if (!e->isRet) {
|
||||
auto event = std::make_unique<Event>(line);
|
||||
if (!event->empty()) {
|
||||
string id = event->id();
|
||||
// insert to table if it is not a function return
|
||||
if (!event->isRet) {
|
||||
auto it = table.find(id);
|
||||
if (it != table.end()) {
|
||||
cout << "Alarm, double event found:\n" << line << std::endl;
|
||||
} else {
|
||||
table.insert(make_pair(id, e));
|
||||
table.insert(make_pair(id, std::move(event)));
|
||||
}
|
||||
// update duration in table
|
||||
} else {
|
||||
auto it = table.find(id);
|
||||
if (it == table.end()) {
|
||||
cout << "Return for unknown event found:\n" << line << std::endl;
|
||||
} else {
|
||||
Event* s = it->second;
|
||||
unique_ptr<Event> ev = std::move(it->second);
|
||||
table.erase(it);
|
||||
s->duration = e->startTime - s->startTime;
|
||||
list.push_back(s);
|
||||
ev->duration = event->startTime - ev->startTime;
|
||||
list.push_back(std::move(ev));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -109,13 +109,11 @@ int main(int argc, char* argv[]) {
|
|||
cout << "Unreturned events:\n";
|
||||
for (auto& p : table) {
|
||||
cout << p.second->pretty() << "\n";
|
||||
delete p.second;
|
||||
}
|
||||
sort(list.begin(), list.end(), [](Event* a, Event* b) -> bool {
|
||||
return *a < *b;
|
||||
});
|
||||
sort(list.begin(), list.end(),
|
||||
[](unique_ptr<Event>const& a, unique_ptr<Event>const& b) -> bool { return *a < *b; });
|
||||
cout << "Events sorted by name and time:\n";
|
||||
for (auto* e : list) {
|
||||
for (auto& e : list) {
|
||||
cout << e->pretty() << "\n";
|
||||
}
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -6,60 +6,78 @@
|
|||
#
|
||||
# This script sets up performance monitoring events to measure single
|
||||
# document operations. Run this script with sudo when the ArangoDB
|
||||
# process is already running. Then do
|
||||
# process is already running:
|
||||
#
|
||||
# ./setupPerfEvents.sh
|
||||
#
|
||||
# Now you are able to recrod the event with:
|
||||
#
|
||||
# sudo perf record -e "probe_arangod:*" -aR sleep 60
|
||||
# (to sample for 60 seconds). A file "perf.data" is written to the
|
||||
# current directory.
|
||||
# Dump the events in this file with
|
||||
#
|
||||
# The above command will get sample data for 60 seconds. A file "perf.data" is
|
||||
# written to the current directory. Dump the events in this file with:
|
||||
#
|
||||
# sudo perf script > perf.history
|
||||
#
|
||||
# This logs the times when individual threads hit the events.
|
||||
# Use the program perfanalyis.cpp in this directory in the following way:
|
||||
#
|
||||
# sudo ./perfanalyis < perf.history > perf.statistics
|
||||
# This will group enter and exit events of functions together, compute
|
||||
# the time spent and sort by function.
|
||||
# Remove all events with
|
||||
#
|
||||
# This will group enter and exit events of functions together, compute the time
|
||||
# spent and sort by function. When finised remove all events with:
|
||||
#
|
||||
# sudo perf probe -d "probe_arangod:*"
|
||||
# List events with
|
||||
#
|
||||
# List events with:
|
||||
#
|
||||
# sudo perf probe -l
|
||||
#
|
||||
#
|
||||
|
||||
ARANGOD_EXECUTABLE=build/bin/arangod
|
||||
perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*"
|
||||
main(){
|
||||
local ARANGOD_EXECUTABLE=${1-build/bin/arangod}
|
||||
|
||||
echo Adding events, this takes a few seconds...
|
||||
#delete all existing events
|
||||
perf probe -x $ARANGOD_EXECUTABLE -d "probe_arangod:*"
|
||||
|
||||
echo "Adding events, this takes a few seconds..."
|
||||
|
||||
echo "Single document operations..."
|
||||
addEvent insertLocal
|
||||
addEvent removeLocal
|
||||
addEvent modifyLocal
|
||||
addEvent documentLocal
|
||||
|
||||
echo "Single document operations on coordinator..."
|
||||
addEvent insertCoordinator
|
||||
addEvent removeCoordinator
|
||||
addEvent updateCoordinator
|
||||
addEvent replaceCoordinator
|
||||
addEvent documentCoordinator
|
||||
|
||||
echo "work method in HttpServerJob"
|
||||
addEvent workHttpServerJob work@HttpServerJob.cpp
|
||||
|
||||
echo "work method in RestDocumentHandler"
|
||||
addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp
|
||||
addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp
|
||||
addEvent handleRequest handleRequest@HttpServer.cpp
|
||||
addEvent handleWrite handleWrite@SocketTask.cpp
|
||||
|
||||
addEvent tcp_sendmsg
|
||||
addEvent tcp_recvmsg
|
||||
|
||||
echo Done.
|
||||
}
|
||||
|
||||
addEvent() {
|
||||
x=$1
|
||||
y=$2
|
||||
if [ "x$y" == "x" ] ; then
|
||||
y=$x
|
||||
fi
|
||||
echo $x
|
||||
perf probe -x $ARANGOD_EXECUTABLE -a $x=$y 2> /dev/null
|
||||
perf probe -x $ARANGOD_EXECUTABLE -a ${x}Ret=$y%return 2> /dev/null
|
||||
local name="$1"
|
||||
local func="${2-"${name}"}"
|
||||
|
||||
echo "setting up $name for function: $func"
|
||||
perf probe -x $ARANGOD_EXECUTABLE -a $name=$func 2> /dev/null #enter function
|
||||
perf probe -x $ARANGOD_EXECUTABLE -a ${name}Ret=$func%return 2> /dev/null #return form function
|
||||
}
|
||||
echo Single document operations...
|
||||
addEvent insertLocal
|
||||
addEvent removeLocal
|
||||
addEvent modifyLocal
|
||||
addEvent documentLocal
|
||||
|
||||
echo Single document operations on coordinator...
|
||||
addEvent insertCoordinator
|
||||
addEvent removeCoordinator
|
||||
addEvent updateCoordinator
|
||||
addEvent replaceCoordinator
|
||||
addEvent documentCoordinator
|
||||
|
||||
echo work method in HttpServerJob
|
||||
addEvent workHttpServerJob work@HttpServerJob.cpp
|
||||
|
||||
echo work method in RestDocumentHandler
|
||||
addEvent executeRestReadDocument readDocument@RestDocumentHandler.cpp
|
||||
addEvent executeRestInsertDocument createDocument@RestDocumentHandler.cpp
|
||||
addEvent handleRequest handleRequest@HttpServer.cpp
|
||||
addEvent handleWrite handleWrite@SocketTask.cpp
|
||||
|
||||
addEvent tcp_sendmsg
|
||||
addEvent tcp_recvmsg
|
||||
|
||||
echo Done.
|
||||
main "$@"
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ start() {
|
|||
PORT=$2
|
||||
mkdir cluster/data$PORT
|
||||
echo Starting $TYPE on port $PORT
|
||||
mkdir -p cluster/apps$PORT
|
||||
build/bin/arangod -c none \
|
||||
--database.directory cluster/data$PORT \
|
||||
--cluster.agency-endpoint tcp://127.0.0.1:$BASE \
|
||||
|
|
|
|||
Loading…
Reference in New Issue