Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

3.5 -- intermediate commit stats (#9780) 

* count intermediate commits

add transactions statistics struct

add tests

put values in object / split test is multiple smaller tests

* fix broken condition

* fix test

* remove print

* fix windows build

* fix mmfiles

* fix cluster

* add structs in docublocks

* we do not know about intermediate commits on coordinators

* add documentation

* add isCluster function

* add serverStatistics to 3.5 client

* update documentation
This commit is contained in:
Jan Christoph Uhde 2019-08-26 15:56:21 +02:00 committed by KVS85
parent 6d53c1e02f
commit 44ab3f04b5
15 changed files with 280 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG
View File

@ -1,6 +1,9 @@
v3.5.1 (XXXX-XX-XX)
-------------------
* Add TransactionStatistics to ServerStatistics (transactions started /
aborted / committed and number of intermediate commits).
* Upgraded version of bundled curl library to 7.65.3.
* Don't retry persisting follower information for collections/shards already

25
Documentation/DocuBlocks/Rest/Administration/get_admin_statistics.md
View File

@ -16,6 +16,18 @@ In case of a distribution, the returned object contains the total count in
*count* and the distribution list in *counts*. The sum (or total) of the
individual values is returned in *sum*.
The transaction statistics show the local started, committed and aborted
transactions as well as intermediate commits done for the server queried. The
intermediate commit count will only take non zero values for the RocksDB
storage engine. Coordinators do almost no local transactions themselves in
their local databases, therefor cluster transactions (transactions started on a
coordinator that require DBServers to finish before the transactions is
committed cluster wide) are just added to their local statistics. This means
that the statistics you would see for a single server is roughly what you can
expect in a cluster setup using a single coordinator querying this coordinator.
Just with the difference that cluster transactions have no notion of
intermediate commits and will not increase the value.
@RESTRETURNCODES
@RESTRETURNCODE{200}
@ -149,7 +161,20 @@ time the server is up and running
@RESTSTRUCT{physicalMemory,server_statistics_struct,integer,required,}
available physical memory on the server
@RESTSTRUCT{transactions,server_statistics_struct,object,required,transactions_struct}
Statistics about transactions
@RESTSTRUCT{started,transactions_struct,integer,required,}
the number of started transactions
@RESTSTRUCT{committed,transactions_struct,integer,required,}
the number of committed transactions
@RESTSTRUCT{aborted,transactions_struct,integer,required,}
the number of aborted transactions
@RESTSTRUCT{intermediateCommits,transactions_struct,integer,required,}
the number of intermediate commits done
@RESTSTRUCT{v8Context,server_statistics_struct,object,required,v8_context_struct}
Statistics about the V8 javascript contexts

21
arangod/ClusterEngine/ClusterTransactionState.cpp
View File

@ -27,6 +27,7 @@
#include "Cluster/ClusterMethods.h"
#include "Cluster/ClusterTrxMethods.h"
#include "ClusterEngine/ClusterEngine.h"
#include "Statistics/ServerStatistics.h"
#include "StorageEngine/EngineSelectorFeature.h"
#include "StorageEngine/TransactionCollection.h"
#include "Transaction/Manager.h"
@ -54,31 +55,33 @@ Result ClusterTransactionState::beginTransaction(transaction::Hints hints) {
// set hints
_hints = hints;
}
auto cleanup = scopeGuard([&] {
if (nestingLevel() == 0) {
updateStatus(transaction::Status::ABORTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsAborted++;
}
// free what we have got so far
unuseCollections(nestingLevel());
});
Result res = useCollections(nestingLevel());
if (res.fail()) { // something is wrong
return res;
}
// all valid
if (nestingLevel() == 0) {
updateStatus(transaction::Status::RUNNING);
transaction::ManagerFeature::manager()->registerTransaction(id(), nullptr);
ServerStatistics::statistics()._transactionsStatistics._transactionsStarted++;
setRegistered();
ClusterEngine* ce = static_cast<ClusterEngine*>(EngineSelectorFeature::ENGINE);
if (ce->isMMFiles() && hasHint(transaction::Hints::Hint::GLOBAL_MANAGED)) {
TRI_ASSERT(isCoordinator());
std::vector<std::string> leaders;
allCollections([&leaders](TransactionCollection& c) {
auto shardIds = c.collection()->shardIds();
@ -90,7 +93,7 @@ Result ClusterTransactionState::beginTransaction(transaction::Hints hints) {
}
return true; // continue
});
res = ClusterTrxMethods::beginTransactionOnLeaders(*this, leaders);
if (res.fail()) { // something is wrong
return res;
@ -99,7 +102,7 @@ Result ClusterTransactionState::beginTransaction(transaction::Hints hints) {
} else {
TRI_ASSERT(_status == transaction::Status::RUNNING);
}
cleanup.cancel();
return res;
}
@ -117,6 +120,7 @@ Result ClusterTransactionState::commitTransaction(transaction::Methods* activeTr
arangodb::Result res;
if (nestingLevel() == 0) {
updateStatus(transaction::Status::COMMITTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsCommitted++;
}
unuseCollections(nestingLevel());
@ -130,6 +134,7 @@ Result ClusterTransactionState::abortTransaction(transaction::Methods* activeTrx
Result res;
if (nestingLevel() == 0) {
updateStatus(transaction::Status::ABORTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsAborted++;
}
unuseCollections(nestingLevel());

7
arangod/MMFiles/MMFilesTransactionState.cpp
View File

@ -36,6 +36,7 @@
#include "StorageEngine/EngineSelectorFeature.h"
#include "StorageEngine/StorageEngine.h"
#include "StorageEngine/TransactionCollection.h"
#include "Statistics/ServerStatistics.h"
#include "Transaction/Methods.h"
#include "VocBase/LogicalCollection.h"
#include "VocBase/ticks.h"
@ -125,19 +126,21 @@ Result MMFilesTransactionState::beginTransaction(transaction::Hints hints) {
// all valid
if (nestingLevel() == 0) {
updateStatus(transaction::Status::RUNNING);
ServerStatistics::statistics()._transactionsStatistics._transactionsStarted++;
// defer writing of the begin marker until necessary!
}
} else {
// something is wrong
if (nestingLevel() == 0) {
updateStatus(transaction::Status::ABORTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsAborted++;
}
// free what we have got so far
unuseCollections(nestingLevel());
}
return result;
}
@ -171,6 +174,7 @@ Result MMFilesTransactionState::commitTransaction(transaction::Methods* activeTr
}
updateStatus(transaction::Status::COMMITTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsCommitted++;
// if a write query, clear the query cache for the participating collections
if (AccessMode::isWriteOrExclusive(_type) && !_collections.empty() &&
@ -199,6 +203,7 @@ Result MMFilesTransactionState::abortTransaction(transaction::Methods* activeTrx
result.reset(res);
updateStatus(transaction::Status::ABORTED);
ServerStatistics::statistics()._transactionsStatistics._transactionsAborted++;
if (_hasOperations) {
// must clean up the query cache because the transaction

8
arangod/RocksDBEngine/RocksDBTransactionState.cpp
View File

@ -39,6 +39,7 @@
#include "StorageEngine/EngineSelectorFeature.h"
#include "StorageEngine/StorageEngine.h"
#include "StorageEngine/TransactionCollection.h"
#include "Statistics/ServerStatistics.h"
#include "Transaction/Context.h"
#include "Transaction/Manager.h"
#include "Transaction/ManagerFeature.h"
@ -102,6 +103,7 @@ Result RocksDBTransactionState::beginTransaction(transaction::Hints hints) {
// register with manager
transaction::ManagerFeature::manager()->registerTransaction(id(), nullptr);
updateStatus(transaction::Status::RUNNING);
ServerStatistics::statistics()._transactionsStatistics._transactionsStarted++;
setRegistered();
@ -188,7 +190,7 @@ void RocksDBTransactionState::createTransaction() {
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
rocksdb::TransactionOptions trxOpts;
trxOpts.set_snapshot = true;
// unclear performance implications do not use for now
// trxOpts.deadlock_detect = !hasHint(transaction::Hints::Hint::NO_DLD);
if (isOnlyExclusiveTransaction()) {
@ -336,7 +338,6 @@ arangodb::Result RocksDBTransactionState::internalCommit() {
coll->commitCounts(id(), postCommitSeq);
committed = true;
}
#ifndef _WIN32
// wait for sync if required, for all other platforms but Windows
if (waitForSync()) {
@ -391,6 +392,7 @@ Result RocksDBTransactionState::commitTransaction(transaction::Methods* activeTr
if (res.ok()) {
updateStatus(transaction::Status::COMMITTED);
cleanupTransaction(); // deletes trx
ServerStatistics::statistics()._transactionsStatistics._transactionsCommitted++;
} else {
abortTransaction(activeTrx); // deletes trx
}
@ -422,6 +424,7 @@ Result RocksDBTransactionState::abortTransaction(transaction::Methods* activeTrx
TRI_ASSERT(!_rocksTransaction && !_cacheTx && !_readSnapshot);
}
ServerStatistics::statistics()._transactionsStatistics._transactionsAborted++;
unuseCollections(nestingLevel());
return result;
}
@ -593,6 +596,7 @@ Result RocksDBTransactionState::triggerIntermediateCommit(bool& hasPerformedInte
}
hasPerformedIntermediateCommit = true;
ServerStatistics::statistics()._transactionsStatistics._intermediateCommits++;
TRI_IF_FAILURE("FailAfterIntermediateCommit") {
THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);

9
arangod/Statistics/Descriptions.cpp
View File

@ -351,10 +351,17 @@ stats::Descriptions::Descriptions()
}
void stats::Descriptions::serverStatistics(velocypack::Builder& b) const {
ServerStatistics info = ServerStatistics::statistics();
ServerStatistics const& info = ServerStatistics::statistics();
b.add("uptime", VPackValue(info._uptime));
b.add("physicalMemory", VPackValue(TRI_PhysicalMemory));
b.add("transactions", VPackValue(VPackValueType::Object));
b.add("started", VPackValue(info._transactionsStatistics._transactionsStarted));
b.add("aborted", VPackValue(info._transactionsStatistics._transactionsAborted));
b.add("committed", VPackValue(info._transactionsStatistics._transactionsCommitted));
b.add("intermediateCommits", VPackValue(info._transactionsStatistics._intermediateCommits));
b.close();
V8DealerFeature* dealer =
application_features::ApplicationServer::getFeature<V8DealerFeature>(
"V8Dealer");

17
arangod/Statistics/ServerStatistics.cpp
View File

@ -30,19 +30,18 @@ using namespace arangodb;
// --SECTION-- static members
// -----------------------------------------------------------------------------
double ServerStatistics::START_TIME = 0.0;
ServerStatistics serverStatisticsGlobal(0);
// -----------------------------------------------------------------------------
// --SECTION-- static public methods
// -----------------------------------------------------------------------------
ServerStatistics ServerStatistics::statistics() {
ServerStatistics server;
server._startTime = START_TIME;
server._uptime = StatisticsFeature::time() - START_TIME;
return server;
ServerStatistics& ServerStatistics::statistics() {
//update the uptime for everyone reading the statistics.
serverStatisticsGlobal._uptime = StatisticsFeature::time() - serverStatisticsGlobal._startTime;
return serverStatisticsGlobal;
}
void ServerStatistics::initialize() { START_TIME = StatisticsFeature::time(); }
void ServerStatistics::initialize(double startTime) {
serverStatisticsGlobal._startTime = startTime;
}

33
arangod/Statistics/ServerStatistics.h
View File

@ -24,18 +24,35 @@
#ifndef ARANGOD_STATISTICS_SERVER_STATISTICS_H
#define ARANGOD_STATISTICS_SERVER_STATISTICS_H 1
class ServerStatistics {
public:
static ServerStatistics statistics();
#include <atomic>
#include <memory>
static void initialize();
struct TransactionStatistics {
TransactionStatistics() : _transactionsStarted(0), _transactionsAborted(0)
, _transactionsCommitted(0), _intermediateCommits(0) {};
std::atomic<std::uint64_t> _transactionsStarted;
std::atomic<std::uint64_t> _transactionsAborted;
std::atomic<std::uint64_t> _transactionsCommitted;
std::atomic<std::uint64_t> _intermediateCommits;
};
struct ServerStatistics {
ServerStatistics(ServerStatistics const&) = delete;
ServerStatistics(ServerStatistics &&) = delete;
ServerStatistics& operator=(ServerStatistics const&) = delete;
ServerStatistics& operator=(ServerStatistics &&) = delete;
static ServerStatistics& statistics();
static void initialize(double);
TransactionStatistics _transactionsStatistics;
double _startTime;
double _uptime;
std::atomic<double> _uptime;
private:
static double START_TIME;
ServerStatistics() : _startTime(0.0), _uptime(0.0) {}
ServerStatistics(double start) : _transactionsStatistics(), _startTime(start), _uptime(0.0) {}
};
#endif

2
arangod/Statistics/StatisticsFeature.cpp
View File

@ -155,7 +155,7 @@ void StatisticsFeature::prepare() {
STATISTICS = this;
ServerStatistics::initialize();
ServerStatistics::initialize(StatisticsFeature::time());
ConnectionStatistics::initialize();
RequestStatistics::initialize();
}

10
arangod/Statistics/StatisticsWorker.cpp
View File

@ -840,7 +840,7 @@ void StatisticsWorker::generateRawStatistics(VPackBuilder& builder, double const
RequestStatistics::fill(totalTime, requestTime, queueTime, ioTime, bytesSent, bytesReceived);
ServerStatistics serverInfo = ServerStatistics::statistics();
ServerStatistics const& serverInfo = ServerStatistics::statistics();
V8DealerFeature* dealer =
application_features::ApplicationServer::getFeature<V8DealerFeature>(
@ -922,6 +922,12 @@ void StatisticsWorker::generateRawStatistics(VPackBuilder& builder, double const
builder.add("server", VPackValue(VPackValueType::Object));
builder.add("uptime", VPackValue(serverInfo._uptime));
builder.add("physicalMemory", VPackValue(TRI_PhysicalMemory));
builder.add("transactions", VPackValue(VPackValueType::Object));
builder.add("started", VPackValue(serverInfo._transactionsStatistics._transactionsStarted));
builder.add("aborted", VPackValue(serverInfo._transactionsStatistics._transactionsAborted));
builder.add("committed", VPackValue(serverInfo._transactionsStatistics._transactionsCommitted));
builder.add("intermediateCommits", VPackValue(serverInfo._transactionsStatistics._intermediateCommits));
builder.close();
// export v8 statistics
builder.add("v8Context", VPackValue(VPackValueType::Object));
@ -958,7 +964,7 @@ void StatisticsWorker::generateRawStatistics(VPackBuilder& builder, double const
builder.add("threads", VPackValue(VPackValueType::Object));
SchedulerFeature::SCHEDULER->toVelocyPack(builder);
builder.close();
// export ttl statistics
TtlFeature* ttlFeature =
application_features::ApplicationServer::getFeature<TtlFeature>("Ttl");

21
arangod/V8Server/v8-statistics.cpp
View File

@ -98,7 +98,7 @@ static void JS_ServerStatistics(v8::FunctionCallbackInfo<v8::Value> const& args)
TRI_V8_TRY_CATCH_BEGIN(isolate)
v8::HandleScope scope(isolate);
ServerStatistics info = ServerStatistics::statistics();
ServerStatistics const& info = ServerStatistics::statistics();
v8::Handle<v8::Object> result = v8::Object::New(isolate);
@ -107,12 +107,25 @@ static void JS_ServerStatistics(v8::FunctionCallbackInfo<v8::Value> const& args)
result->Set(TRI_V8_ASCII_STRING(isolate, "physicalMemory"),
v8::Number::New(isolate, (double)TRI_PhysicalMemory));
v8::Handle<v8::Object> v8CountersObj = v8::Object::New(isolate);
// transaction info
auto const& ts = info._transactionsStatistics;
v8::Handle<v8::Object> v8TransactionInfoObj = v8::Object::New(isolate);
v8TransactionInfoObj->Set(TRI_V8_ASCII_STRING(isolate, "started"),
v8::Number::New(isolate, (double)ts._transactionsStarted));
v8TransactionInfoObj->Set(TRI_V8_ASCII_STRING(isolate, "aborted"),
v8::Number::New(isolate, (double)ts._transactionsAborted));
v8TransactionInfoObj->Set(TRI_V8_ASCII_STRING(isolate, "committed"),
v8::Number::New(isolate, (double)ts._transactionsCommitted));
v8TransactionInfoObj->Set(TRI_V8_ASCII_STRING(isolate, "intermediateCommits"),
v8::Number::New(isolate, (double)ts._intermediateCommits));
result->Set(TRI_V8_ASCII_STRING(isolate, "transactions"), v8TransactionInfoObj);
// v8 counters
V8DealerFeature* dealer =
application_features::ApplicationServer::getFeature<V8DealerFeature>(
"V8Dealer");
auto v8Counters = dealer->getCurrentContextNumbers();
v8::Handle<v8::Object> v8CountersObj = v8::Object::New(isolate);
v8CountersObj->Set(TRI_V8_ASCII_STRING(isolate, "available"),
v8::Number::New(isolate, static_cast<int32_t>(v8Counters.available)));
v8CountersObj->Set(TRI_V8_ASCII_STRING(isolate, "busy"),
@ -146,7 +159,7 @@ static void JS_ServerStatistics(v8::FunctionCallbackInfo<v8::Value> const& args)
v8CountersObj->Set(TRI_V8_ASCII_STRING(isolate, "memory"),
v8ListOfMemory);
result->Set(TRI_V8_ASCII_STRING(isolate, "v8Context"), v8CountersObj);
v8::Handle<v8::Object> counters = v8::Object::New(isolate);

16
js/client/bootstrap/modules/internal.js
View File

@ -334,6 +334,22 @@
exports.output(level, ': ', msg, '\n');
};
exports.isCluster = function () {
const arangosh = require('@arangodb/arangosh');
let requestResult = exports.arango.GET("/_admin/server/role");
arangosh.checkRequestResult(requestResult);
return requestResult.role === "COORDINATOR";
};
// / @brief serverStatistics
exports.serverStatistics = function () {
const arangosh = require('@arangodb/arangosh');
let requestResult = exports.arango.GET('/_admin/statistics');
arangosh.checkRequestResult(requestResult);
return requestResult.server;
};
// //////////////////////////////////////////////////////////////////////////////
// / @brief sprintf wrapper
// //////////////////////////////////////////////////////////////////////////////

9
js/server/bootstrap/modules/internal.js
View File

@ -526,5 +526,14 @@
exports.debugCanUseFailAt = global.SYS_DEBUG_CAN_USE_FAILAT;
delete global.SYS_DEBUG_CAN_USE_FAILAT;
}
// /////////////////////////////////////////////////////////////////////////////
// / @brief whether or not clustering is enabled
// /////////////////////////////////////////////////////////////////////////////
exports.isCluster = function () {
var role = global.ArangoServerState.role();
return (role !== undefined && role !== 'SINGLE' && role !== 'AGENT');
};
}());

2
tests/js/client/shell/shell-transaction.js
View File

@ -26,6 +26,8 @@
// / @author Simon Grätzer
// //////////////////////////////////////////////////////////////////////////////
// tests for streaming transactions
var jsunity = require('jsunity');
var internal = require('internal');
var arangodb = require('@arangodb');

133
tests/js/common/shell/shell-statistics.js Normal file
View File

@ -0,0 +1,133 @@
/*jshint globalstrict:false, strict:false */
/*global assertEqual, assertTrue, assertMatch, fail */
////////////////////////////////////////////////////////////////////////////////
/// @brief tests for client/server side transaction invocation
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2019 ArangoDB Inc, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Christoph Uhde
/// @author Copyright 2019, ArangodDB Inc, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
const jsunity = require("jsunity");
const internal = require("internal");
const db = require("@arangodb").db;
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////
function CommonStatisticsSuite() {
'use strict';
let c;
return {
setUp: function () {
c = db._create("shellCommonStatTestCollection");
},
tearDown: function () {
db._drop("shellCommonStatTestCollection");
},
testServerStatsStructure: function () {
let stats = internal.serverStatistics();
assertTrue(Number.isInteger(stats.transactions.started));
assertTrue(Number.isInteger(stats.transactions.committed));
assertTrue(Number.isInteger(stats.transactions.aborted));
assertTrue(Number.isInteger(stats.transactions.intermediateCommits));
},
testServerStatsCommit: function () {
let stats1 = internal.serverStatistics();
c.insert({ "gondel" : "ulf" });
let stats2 = internal.serverStatistics();
assertTrue(stats1.transactions.started < stats2.transactions.started
, "1 started: " + stats1.transactions.started
+ " -- 2 started: " + stats2.transactions.started);
assertTrue(stats1.transactions.committed < stats2.transactions.committed);
},
testServerStatsAbort: function () {
let stats1 = internal.serverStatistics();
let stats2;
try {
db._executeTransaction({
collections: {},
action: function () {
var err = new Error('abort on purpose');
throw err;
}
});
fail();
} catch (err) {
stats2 = internal.serverStatistics();
assertMatch(/abort on purpose/, err.errorMessage);
}
assertTrue(stats1.transactions.started < stats2.transactions.started);
assertTrue(stats1.transactions.aborted < stats2.transactions.aborted);
},
testIntermediateCommitsCommit: function () {
let stats1 = internal.serverStatistics();
db._query(`FOR i IN 1..3 INSERT { "ulf" : i } IN ${c.name()}`, {}, { "intermediateCommitCount" : 2});
let stats2 = internal.serverStatistics();
if(db._engine().name === "rocksdb" && !internal.isCluster()) {
assertTrue(stats1.transactions.intermediateCommits < stats2.transactions.intermediateCommits);
} else {
assertEqual(stats1.transactions.intermediateCommits, 0);
}
assertTrue(stats1.transactions.committed < stats2.transactions.committed);
},
testIntermediateCommitsAbort: function () {
let stats1 = internal.serverStatistics();
let stats2;
try {
let rv = db._query(`FOR i IN 1..3
FILTER ASSERT(i == 0, "abort on purpose")
INSERT { "ulf" : i } IN ${c.name()}
`, {}, { "intermediateCommitCount" : 2});
fail();
} catch (err) {
stats2 = internal.serverStatistics();
assertMatch(/abort on purpose/, err.errorMessage);
}
if(db._engine().name === "rocksdb" && !internal.isCluster()) {
assertTrue(stats1.transactions.intermediateCommits <= stats2.transactions.intermediateCommits);
} else {
assertEqual(stats1.transactions.intermediateCommits, 0);
}
assertTrue(stats1.transactions.aborted < stats2.transactions.aborted);
},
}; //return
} // CommonStatisticsSuite end
jsunity.run(CommonStatisticsSuite);
return jsunity.done();