From 8312c64f6021461e586d73beb167cab4dd0cee7b Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 18:21:06 +0200 Subject: [PATCH 01/10] Unwurst results --- UnitTests/unittest.js | 101 +++++++--------------- js/client/modules/@arangodb/testing.js | 112 +++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 70 deletions(-) diff --git a/UnitTests/unittest.js b/UnitTests/unittest.js index c472fafc39..31aef39460 100644 --- a/UnitTests/unittest.js +++ b/UnitTests/unittest.js @@ -71,78 +71,39 @@ function resultsToXml(results, baseName, cluster) { clprefix = 'CL_'; } - const isSignificant = function(a, b) { - return (internalMembers.indexOf(b) === -1) && a.hasOwnProperty(b); - }; + let cleanedResults = UnitTest.unwurst(results); + print(JSON.stringify(cleanedResults)); + cleanedResults.forEach(suite => { + print(suite.suiteName); + let xml = buildXml(); + xml.elem('testsuite', { + errors: suite.tests.filter(test => test.hasOwnProperty('error')).length, + failures: suite.tests.filter(test => test.hasOwnProperty('failure')).length, + tests: suite.tests.length, + name: suite.suiteName, + }); - for (let resultName in results) { - if (isSignificant(results, resultName)) { - let run = results[resultName]; - - for (let runName in run) { - if (isSignificant(run, runName)) { - const xmlName = clprefix + resultName + "_" + runName; - const current = run[runName]; - - if (current.skipped) { - continue; - } - - let xml = buildXml(); - let total = 0; - - if (current.hasOwnProperty('total')) { - total = current.total; - } - - let failuresFound = current.failed; - xml.elem("testsuite", { - errors: 0, - failures: failuresFound, - tests: total, - name: xmlName, - time: 0 + current.duration - }); - - let seen = false; - - for (let oneTestName in current) { - if (isSignificant(current, oneTestName)) { - const oneTest = current[oneTestName]; - const success = (oneTest.status === true); - - seen = true; - - xml.elem("testcase", { - name: clprefix + oneTestName, - time: 0 + oneTest.duration - }, success); - - if (!success) { - xml.elem("failure"); - xml.text('\n'); - xml.elem("/failure"); - xml.elem("/testcase"); - } - } - } - - if (!seen) { - xml.elem("testcase", { - name: 'all_tests_in_' + xmlName, - time: 0 + current.duration - }, true); - } - - xml.elem("/testsuite"); - - const fn = makePathGeneric(baseName + xmlName + ".xml").join('_'); - - fs.write(testOutputDirectory + fn, xml.join("")); - } + suite.tests.forEach(test => { + xml.elem('testcase', { + name: test.testName, + }); + if (test.error) { + xml.elem('error'); + xml.text('\n'); + xml.elem('/error'); } - } - } + if (test.failure) { + xml.elem('failure'); + xml.text('\n'); + xml.elem('/failure'); + } + xml.elem('/testcase'); + }); + xml.elem('/testsuite'); + + const fn = makePathGeneric(baseName + suite.suiteName + ".xml").join('_'); + fs.write(testOutputDirectory + fn, xml.join("")); + }); } //////////////////////////////////////////////////////////////////////////////// diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index dfe90fa661..bec6fef145 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -198,6 +198,117 @@ function testCaseMessage (test) { } } +function unwurst(r) { + function skipInternalMember (r, a) { + return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; + } + + let testSuites = []; + /* jshint forin: false */ + for (let testrunName in r) { + if (skipInternalMember(r, testrunName)) { + continue; + } + + let testrun = r[testrunName]; + + let successCases = {}; + let failedCases = {}; + let isSuccess = true; + + let suiteDefinition = { + suiteName: testrunName, + tests: [], + } + + for (let testName in testrun) { + if (skipInternalMember(testrun, testName)) { + continue; + } + + let test = testrun[testName]; + + if (test.status) { + successCases[testName] = test; + } else { + isSuccess = false; + ++failedSuite; + + if (test.hasOwnProperty('message')) { + ++failedTests; + failedCases[testName] = { + test: testCaseMessage(test) + }; + } else { + let fails = failedCases[testName] = {}; + + for (let oneName in test) { + if (skipInternalMember(test, oneName)) { + continue; + } + + let oneTest = test[oneName]; + + if (!oneTest.status) { + ++failedTests; + fails[oneName] = testCaseMessage(oneTest); + } + } + } + } + } + + for (let name in successCases) { + if (!successCases.hasOwnProperty(name)) { + continue; + } + let thisTest = { + testName: name, + } + let details = successCases[name]; + + if (details.skipped) { + thisTest.skipped = true; + } + + suiteDefinition.tests.push(thisTest); + } + + for (let name in failedCases) { + if (!failedCases.hasOwnProperty(name)) { + continue; + } + + let details = failedCases[name]; + let message = ''; + for (let one in details) { + if (!details.hasOwnProperty(one)) { + continue; + } + message += details[one]; + } + + suiteDefinition.tests.push({ + testName: name, + failure: message, + }); + } + testSuites.push(suiteDefinition); + } + + let arangodStatusBogusTest = { + testName: 'arangod', + } + if (r.crashed) { + arangodStatusBogusTest.error = 'arangod crashed during execution!'; + } + testSuites.push({ + suiteName: 'arangod-status', + tests: [arangodStatusBogusTest], + }); + return testSuites; +} + function unitTestPrettyPrintResults (r, testOutputDirectory, options) { function skipInternalMember (r, a) { return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; @@ -546,6 +657,7 @@ function unitTest (cases, options) { exports.unitTest = unitTest; exports.internalMembers = internalMembers; +exports.unwurst = unwurst; exports.testFuncs = testFuncs; exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults; From 59df82c975aa3492dbdf2bb75dd7c48d0e33bde4 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 18:26:27 +0200 Subject: [PATCH 02/10] Fail lint --- js/client/modules/@arangodb/testing.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index bec6fef145..5bb06c4ef9 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -219,7 +219,7 @@ function unwurst(r) { let suiteDefinition = { suiteName: testrunName, tests: [], - } + }; for (let testName in testrun) { if (skipInternalMember(testrun, testName)) { @@ -232,10 +232,8 @@ function unwurst(r) { successCases[testName] = test; } else { isSuccess = false; - ++failedSuite; if (test.hasOwnProperty('message')) { - ++failedTests; failedCases[testName] = { test: testCaseMessage(test) }; @@ -250,7 +248,6 @@ function unwurst(r) { let oneTest = test[oneName]; if (!oneTest.status) { - ++failedTests; fails[oneName] = testCaseMessage(oneTest); } } @@ -264,7 +261,7 @@ function unwurst(r) { } let thisTest = { testName: name, - } + }; let details = successCases[name]; if (details.skipped) { @@ -298,7 +295,7 @@ function unwurst(r) { let arangodStatusBogusTest = { testName: 'arangod', - } + }; if (r.crashed) { arangodStatusBogusTest.error = 'arangod crashed during execution!'; } From 570a5a67b3cf50fad31ef2aa9468d5c420a0188e Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 18:35:51 +0200 Subject: [PATCH 03/10] Remove debug leftover --- UnitTests/unittest.js | 1 - 1 file changed, 1 deletion(-) diff --git a/UnitTests/unittest.js b/UnitTests/unittest.js index 31aef39460..4255b39cec 100644 --- a/UnitTests/unittest.js +++ b/UnitTests/unittest.js @@ -72,7 +72,6 @@ function resultsToXml(results, baseName, cluster) { } let cleanedResults = UnitTest.unwurst(results); - print(JSON.stringify(cleanedResults)); cleanedResults.forEach(suite => { print(suite.suiteName); let xml = buildXml(); From 860dcbbf9ae15546afbd3dc283fd3c70d9f7b443 Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 2 May 2017 13:03:00 -0400 Subject: [PATCH 04/10] Enabled/fixed check for empty transactions to reduce WAL spam. --- .../RocksDBEngine/RocksDBTransactionState.cpp | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBTransactionState.cpp b/arangod/RocksDBEngine/RocksDBTransactionState.cpp index 807cf61537..8beb274bfd 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionState.cpp +++ b/arangod/RocksDBEngine/RocksDBTransactionState.cpp @@ -55,7 +55,9 @@ using namespace arangodb; // for the RocksDB engine we do not need any additional data struct RocksDBTransactionData final : public TransactionData {}; -RocksDBSavePoint::RocksDBSavePoint(rocksdb::Transaction* trx, bool handled, std::function const& rollbackCallback) +RocksDBSavePoint::RocksDBSavePoint( + rocksdb::Transaction* trx, bool handled, + std::function const& rollbackCallback) : _trx(trx), _rollbackCallback(rollbackCallback), _handled(handled) { TRI_ASSERT(trx != nullptr); if (!_handled) { @@ -110,8 +112,8 @@ RocksDBTransactionState::~RocksDBTransactionState() { /// @brief start a transaction Result RocksDBTransactionState::beginTransaction(transaction::Hints hints) { - LOG_TRX(this, _nestingLevel) - << "beginning " << AccessMode::typeString(_type) << " transaction"; + LOG_TRX(this, _nestingLevel) << "beginning " << AccessMode::typeString(_type) + << " transaction"; Result result = useCollections(_nestingLevel); @@ -157,12 +159,13 @@ Result RocksDBTransactionState::beginTransaction(transaction::Hints hints) { _rocksTransaction->SetSnapshot(); _rocksReadOptions.snapshot = _rocksTransaction->GetSnapshot(); - if (!isReadOnlyTransaction() && !hasHint(transaction::Hints::Hint::SINGLE_OPERATION)) { + if (!isReadOnlyTransaction() && + !hasHint(transaction::Hints::Hint::SINGLE_OPERATION)) { RocksDBLogValue header = - RocksDBLogValue::BeginTransaction(_vocbase->id(), _id); + RocksDBLogValue::BeginTransaction(_vocbase->id(), _id); _rocksTransaction->PutLogData(header.slice()); } - + } else { TRI_ASSERT(_status == transaction::Status::RUNNING); } @@ -173,8 +176,8 @@ Result RocksDBTransactionState::beginTransaction(transaction::Hints hints) { /// @brief commit a transaction Result RocksDBTransactionState::commitTransaction( transaction::Methods* activeTrx) { - LOG_TRX(this, _nestingLevel) - << "committing " << AccessMode::typeString(_type) << " transaction"; + LOG_TRX(this, _nestingLevel) << "committing " << AccessMode::typeString(_type) + << " transaction"; TRI_ASSERT(_status == transaction::Status::RUNNING); TRI_IF_FAILURE("TransactionWriteCommitMarker") { @@ -185,13 +188,14 @@ Result RocksDBTransactionState::commitTransaction( if (_nestingLevel == 0) { if (_rocksTransaction != nullptr) { - if (hasOperations()) { - // set wait for sync flag if required + // if (hasOperations()) { + if (_rocksTransaction->GetNumKeys() > 0) { + // set wait for sync flag if required if (waitForSync()) { _rocksWriteOptions.sync = true; _rocksTransaction->SetWriteOptions(_rocksWriteOptions); } - + // TODO wait for response on github issue to see how we can use the // sequence number result = rocksutils::convertStatus(_rocksTransaction->Commit()); @@ -231,10 +235,8 @@ Result RocksDBTransactionState::commitTransaction( } } else { // don't write anything if the transaction is empty - // TODO: calling Rollback() here does not work for some reason but it should. - // must investigate further!! - result = rocksutils::convertStatus(_rocksTransaction->Commit()); - + result = rocksutils::convertStatus(_rocksTransaction->Rollback()); + if (_cacheTx != nullptr) { // note: endTransaction() will delete _cacheTx! CacheManagerFeature::MANAGER->endTransaction(_cacheTx); @@ -256,8 +258,8 @@ Result RocksDBTransactionState::commitTransaction( /// @brief abort and rollback a transaction Result RocksDBTransactionState::abortTransaction( transaction::Methods* activeTrx) { - LOG_TRX(this, _nestingLevel) - << "aborting " << AccessMode::typeString(_type) << " transaction"; + LOG_TRX(this, _nestingLevel) << "aborting " << AccessMode::typeString(_type) + << " transaction"; TRI_ASSERT(_status == transaction::Status::RUNNING); Result result; @@ -265,7 +267,7 @@ Result RocksDBTransactionState::abortTransaction( if (_rocksTransaction != nullptr) { rocksdb::Status status = _rocksTransaction->Rollback(); result = rocksutils::convertStatus(status); - + if (_cacheTx != nullptr) { // note: endTransaction() will delete _cacheTx! CacheManagerFeature::MANAGER->endTransaction(_cacheTx); @@ -290,26 +292,25 @@ Result RocksDBTransactionState::abortTransaction( } void RocksDBTransactionState::prepareOperation( - TRI_voc_cid_t collectionId, TRI_voc_rid_t revisionId, - StringRef const& key, TRI_voc_document_operation_e operationType) { - + TRI_voc_cid_t collectionId, TRI_voc_rid_t revisionId, StringRef const& key, + TRI_voc_document_operation_e operationType) { TRI_ASSERT(!isReadOnlyTransaction()); - + bool singleOp = hasHint(transaction::Hints::Hint::SINGLE_OPERATION); // single operations should never call this method twice - TRI_ASSERT(!singleOp || _lastUsedCollection == 0); + TRI_ASSERT(!singleOp || _lastUsedCollection == 0); if (collectionId != _lastUsedCollection) { switch (operationType) { case TRI_VOC_DOCUMENT_OPERATION_INSERT: case TRI_VOC_DOCUMENT_OPERATION_UPDATE: case TRI_VOC_DOCUMENT_OPERATION_REPLACE: { if (singleOp) { - RocksDBLogValue logValue = RocksDBLogValue::SinglePut(_vocbase->id(), - collectionId); + RocksDBLogValue logValue = + RocksDBLogValue::SinglePut(_vocbase->id(), collectionId); _rocksTransaction->PutLogData(logValue.slice()); } else { RocksDBLogValue logValue = - RocksDBLogValue::DocumentOpsPrologue(collectionId); + RocksDBLogValue::DocumentOpsPrologue(collectionId); _rocksTransaction->PutLogData(logValue.slice()); } break; @@ -317,13 +318,12 @@ void RocksDBTransactionState::prepareOperation( case TRI_VOC_DOCUMENT_OPERATION_REMOVE: { if (singleOp) { TRI_ASSERT(!key.empty()); - RocksDBLogValue logValue = RocksDBLogValue::SingleRemove(_vocbase->id(), - collectionId, - key); + RocksDBLogValue logValue = + RocksDBLogValue::SingleRemove(_vocbase->id(), collectionId, key); _rocksTransaction->PutLogData(logValue.slice()); } else { RocksDBLogValue logValue = - RocksDBLogValue::DocumentOpsPrologue(collectionId); + RocksDBLogValue::DocumentOpsPrologue(collectionId); _rocksTransaction->PutLogData(logValue.slice()); } } break; @@ -332,11 +332,11 @@ void RocksDBTransactionState::prepareOperation( } _lastUsedCollection = collectionId; } - - // we need to log the remove log entry, if we don't have the single optimization + + // we need to log the remove log entry, if we don't have the single + // optimization if (!singleOp && operationType == TRI_VOC_DOCUMENT_OPERATION_REMOVE) { - RocksDBLogValue logValue = - RocksDBLogValue::DocumentRemove(key); + RocksDBLogValue logValue = RocksDBLogValue::DocumentRemove(key); _rocksTransaction->PutLogData(logValue.slice()); } } From 6112caec4a535c73c3924a1bbac1c091a2758c54 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 19:05:27 +0200 Subject: [PATCH 05/10] Revert "Remove debug leftover" This reverts commit 570a5a67b3cf50fad31ef2aa9468d5c420a0188e. --- UnitTests/unittest.js | 1 + 1 file changed, 1 insertion(+) diff --git a/UnitTests/unittest.js b/UnitTests/unittest.js index 4255b39cec..31aef39460 100644 --- a/UnitTests/unittest.js +++ b/UnitTests/unittest.js @@ -72,6 +72,7 @@ function resultsToXml(results, baseName, cluster) { } let cleanedResults = UnitTest.unwurst(results); + print(JSON.stringify(cleanedResults)); cleanedResults.forEach(suite => { print(suite.suiteName); let xml = buildXml(); From 525a9a0d4cf403d78cde74218f577547eca289fe Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 19:05:40 +0200 Subject: [PATCH 06/10] Revert "Fail lint" This reverts commit 59df82c975aa3492dbdf2bb75dd7c48d0e33bde4. --- js/client/modules/@arangodb/testing.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index 5bb06c4ef9..bec6fef145 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -219,7 +219,7 @@ function unwurst(r) { let suiteDefinition = { suiteName: testrunName, tests: [], - }; + } for (let testName in testrun) { if (skipInternalMember(testrun, testName)) { @@ -232,8 +232,10 @@ function unwurst(r) { successCases[testName] = test; } else { isSuccess = false; + ++failedSuite; if (test.hasOwnProperty('message')) { + ++failedTests; failedCases[testName] = { test: testCaseMessage(test) }; @@ -248,6 +250,7 @@ function unwurst(r) { let oneTest = test[oneName]; if (!oneTest.status) { + ++failedTests; fails[oneName] = testCaseMessage(oneTest); } } @@ -261,7 +264,7 @@ function unwurst(r) { } let thisTest = { testName: name, - }; + } let details = successCases[name]; if (details.skipped) { @@ -295,7 +298,7 @@ function unwurst(r) { let arangodStatusBogusTest = { testName: 'arangod', - }; + } if (r.crashed) { arangodStatusBogusTest.error = 'arangod crashed during execution!'; } From 6527c8ceede4dbe92ca5da32f84f686f89e22742 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Tue, 2 May 2017 19:05:48 +0200 Subject: [PATCH 07/10] Revert "Unwurst results" This reverts commit 8312c64f6021461e586d73beb167cab4dd0cee7b. --- UnitTests/unittest.js | 101 +++++++++++++++------- js/client/modules/@arangodb/testing.js | 112 ------------------------- 2 files changed, 70 insertions(+), 143 deletions(-) diff --git a/UnitTests/unittest.js b/UnitTests/unittest.js index 31aef39460..c472fafc39 100644 --- a/UnitTests/unittest.js +++ b/UnitTests/unittest.js @@ -71,39 +71,78 @@ function resultsToXml(results, baseName, cluster) { clprefix = 'CL_'; } - let cleanedResults = UnitTest.unwurst(results); - print(JSON.stringify(cleanedResults)); - cleanedResults.forEach(suite => { - print(suite.suiteName); - let xml = buildXml(); - xml.elem('testsuite', { - errors: suite.tests.filter(test => test.hasOwnProperty('error')).length, - failures: suite.tests.filter(test => test.hasOwnProperty('failure')).length, - tests: suite.tests.length, - name: suite.suiteName, - }); + const isSignificant = function(a, b) { + return (internalMembers.indexOf(b) === -1) && a.hasOwnProperty(b); + }; - suite.tests.forEach(test => { - xml.elem('testcase', { - name: test.testName, - }); - if (test.error) { - xml.elem('error'); - xml.text('\n'); - xml.elem('/error'); - } - if (test.failure) { - xml.elem('failure'); - xml.text('\n'); - xml.elem('/failure'); - } - xml.elem('/testcase'); - }); - xml.elem('/testsuite'); + for (let resultName in results) { + if (isSignificant(results, resultName)) { + let run = results[resultName]; - const fn = makePathGeneric(baseName + suite.suiteName + ".xml").join('_'); - fs.write(testOutputDirectory + fn, xml.join("")); - }); + for (let runName in run) { + if (isSignificant(run, runName)) { + const xmlName = clprefix + resultName + "_" + runName; + const current = run[runName]; + + if (current.skipped) { + continue; + } + + let xml = buildXml(); + let total = 0; + + if (current.hasOwnProperty('total')) { + total = current.total; + } + + let failuresFound = current.failed; + xml.elem("testsuite", { + errors: 0, + failures: failuresFound, + tests: total, + name: xmlName, + time: 0 + current.duration + }); + + let seen = false; + + for (let oneTestName in current) { + if (isSignificant(current, oneTestName)) { + const oneTest = current[oneTestName]; + const success = (oneTest.status === true); + + seen = true; + + xml.elem("testcase", { + name: clprefix + oneTestName, + time: 0 + oneTest.duration + }, success); + + if (!success) { + xml.elem("failure"); + xml.text('\n'); + xml.elem("/failure"); + xml.elem("/testcase"); + } + } + } + + if (!seen) { + xml.elem("testcase", { + name: 'all_tests_in_' + xmlName, + time: 0 + current.duration + }, true); + } + + xml.elem("/testsuite"); + + const fn = makePathGeneric(baseName + xmlName + ".xml").join('_'); + + fs.write(testOutputDirectory + fn, xml.join("")); + } + } + } + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/js/client/modules/@arangodb/testing.js b/js/client/modules/@arangodb/testing.js index bec6fef145..dfe90fa661 100644 --- a/js/client/modules/@arangodb/testing.js +++ b/js/client/modules/@arangodb/testing.js @@ -198,117 +198,6 @@ function testCaseMessage (test) { } } -function unwurst(r) { - function skipInternalMember (r, a) { - return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; - } - - let testSuites = []; - /* jshint forin: false */ - for (let testrunName in r) { - if (skipInternalMember(r, testrunName)) { - continue; - } - - let testrun = r[testrunName]; - - let successCases = {}; - let failedCases = {}; - let isSuccess = true; - - let suiteDefinition = { - suiteName: testrunName, - tests: [], - } - - for (let testName in testrun) { - if (skipInternalMember(testrun, testName)) { - continue; - } - - let test = testrun[testName]; - - if (test.status) { - successCases[testName] = test; - } else { - isSuccess = false; - ++failedSuite; - - if (test.hasOwnProperty('message')) { - ++failedTests; - failedCases[testName] = { - test: testCaseMessage(test) - }; - } else { - let fails = failedCases[testName] = {}; - - for (let oneName in test) { - if (skipInternalMember(test, oneName)) { - continue; - } - - let oneTest = test[oneName]; - - if (!oneTest.status) { - ++failedTests; - fails[oneName] = testCaseMessage(oneTest); - } - } - } - } - } - - for (let name in successCases) { - if (!successCases.hasOwnProperty(name)) { - continue; - } - let thisTest = { - testName: name, - } - let details = successCases[name]; - - if (details.skipped) { - thisTest.skipped = true; - } - - suiteDefinition.tests.push(thisTest); - } - - for (let name in failedCases) { - if (!failedCases.hasOwnProperty(name)) { - continue; - } - - let details = failedCases[name]; - let message = ''; - for (let one in details) { - if (!details.hasOwnProperty(one)) { - continue; - } - message += details[one]; - } - - suiteDefinition.tests.push({ - testName: name, - failure: message, - }); - } - testSuites.push(suiteDefinition); - } - - let arangodStatusBogusTest = { - testName: 'arangod', - } - if (r.crashed) { - arangodStatusBogusTest.error = 'arangod crashed during execution!'; - } - testSuites.push({ - suiteName: 'arangod-status', - tests: [arangodStatusBogusTest], - }); - return testSuites; -} - function unitTestPrettyPrintResults (r, testOutputDirectory, options) { function skipInternalMember (r, a) { return !r.hasOwnProperty(a) || internalMembers.indexOf(a) !== -1; @@ -657,7 +546,6 @@ function unitTest (cases, options) { exports.unitTest = unitTest; exports.internalMembers = internalMembers; -exports.unwurst = unwurst; exports.testFuncs = testFuncs; exports.unitTestPrettyPrintResults = unitTestPrettyPrintResults; From b2e6ca3aa22af04bb9066e1fe85ddeafa7d0d489 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Tue, 2 May 2017 22:36:15 +0200 Subject: [PATCH 08/10] Translate distributeShardsLike into collection name in dump. --- arangod/VocBase/LogicalCollection.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 0e7aa44520..81cc3779d3 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -747,9 +747,22 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result, std::unordered_set ignoreKeys{"allowUserKeys", "cid", "count", "objectId", - "statusString", "version"}; + "statusString", "version", + "distributeShardsLike"}; VPackBuilder params = toVelocyPackIgnore(ignoreKeys, false); - result.add(params.slice()); + { VPackObjectBuilder guard(&result); + for (auto const& p : VPackObjectIterator(params.slice())) { + result.add(p.key); + result.add(p.value); + } + if (!_distributeShardsLike.empty()) { + CollectionNameResolver resolver(_vocbase); + result.add("distributeShardsLike", + VPackValue(resolver.getCollectionNameCluster( + static_cast(basics::StringUtils::uint64( + distributeShardsLike()))))); + } + } result.add(VPackValue("indexes")); getIndexesVPack(result, false); From 57606e3e180f65edacd673b31bd7f639b7f6ddbf Mon Sep 17 00:00:00 2001 From: Dan Larkin Date: Tue, 2 May 2017 17:00:45 -0400 Subject: [PATCH 09/10] Fixed logic bug preventing file deletions in RocksDB engine. --- arangod/RocksDBEngine/RocksDBReplicationManager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBReplicationManager.cpp b/arangod/RocksDBEngine/RocksDBReplicationManager.cpp index 881e6671e7..5a8a448198 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationManager.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationManager.cpp @@ -285,6 +285,8 @@ bool RocksDBReplicationManager::garbageCollect(bool force) { MUTEX_LOCKER(mutexLocker, _lock); + auto oldSize = _contexts.size(); + for (auto it = _contexts.begin(); it != _contexts.end(); /* no hoisting */) { auto context = it->second; @@ -318,7 +320,7 @@ bool RocksDBReplicationManager::garbageCollect(bool force) { // FIXME effectively force should only be called on shutdown // nevertheless this is quite ugly - if (_contexts.size() == 0 && !force) { + if ((oldSize > 0) && (_contexts.size() == 0) && !force) { enableFileDeletions(); } } catch (...) { @@ -341,7 +343,7 @@ void RocksDBReplicationManager::disableFileDeletions() { void RocksDBReplicationManager::enableFileDeletions() { auto rocks = globalRocksDB(); - auto s = rocks->DisableFileDeletions(); + auto s = rocks->EnableFileDeletions(false); TRI_ASSERT(s.ok()); } From 1b0e976a5558bfb866d93deebfde35c38333b7d9 Mon Sep 17 00:00:00 2001 From: Max Neunhoeffer Date: Wed, 3 May 2017 09:13:03 +0200 Subject: [PATCH 10/10] Fix ignoreDistributeShardsLikeErrors behaviour. --- arangod/Cluster/ClusterMethods.h | 8 ++++---- .../RocksDBRestReplicationHandler.cpp | 16 +++++++++++----- .../RocksDBRestReplicationHandler.h | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/arangod/Cluster/ClusterMethods.h b/arangod/Cluster/ClusterMethods.h index 5bdd3b6dcd..28b3c936f5 100644 --- a/arangod/Cluster/ClusterMethods.h +++ b/arangod/Cluster/ClusterMethods.h @@ -258,8 +258,8 @@ class ClusterMethods { static std::unique_ptr createCollectionOnCoordinator( TRI_col_type_e collectionType, TRI_vocbase_t* vocbase, arangodb::velocypack::Slice parameters, - bool ignoreDistributeShardsLikeErrors = true, - bool waitForSyncReplication = true); + bool ignoreDistributeShardsLikeErrors, + bool waitForSyncReplication); private: @@ -268,8 +268,8 @@ class ClusterMethods { //////////////////////////////////////////////////////////////////////////////// static std::unique_ptr persistCollectionInAgency( - LogicalCollection* col, bool ignoreDistributeShardsLikeErrors = true, - bool waitForSyncReplication = true); + LogicalCollection* col, bool ignoreDistributeShardsLikeErrors, + bool waitForSyncReplication); }; } // namespace arangodb diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp index 2f2c890078..752e12cf57 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp @@ -903,6 +903,11 @@ void RocksDBRestReplicationHandler::handleCommandRestoreCollection() { force = StringUtils::boolean(value3); } + std::string const& value9 = + _request->value("ignoreDistributeShardsLikeErrors", found); + bool ignoreDistributeShardsLikeErrors = + found ? StringUtils::boolean(value9) : false; + uint64_t numberOfShards = 0; std::string const& value4 = _request->value("numberOfShards", found); @@ -921,9 +926,9 @@ void RocksDBRestReplicationHandler::handleCommandRestoreCollection() { int res; if (ServerState::instance()->isCoordinator()) { - res = processRestoreCollectionCoordinator(slice, overwrite, recycleIds, - force, numberOfShards, errorMsg, - replicationFactor); + res = processRestoreCollectionCoordinator( + slice, overwrite, recycleIds, force, numberOfShards, errorMsg, + replicationFactor, ignoreDistributeShardsLikeErrors); } else { res = processRestoreCollection(slice, overwrite, recycleIds, force, errorMsg); @@ -2352,7 +2357,7 @@ int RocksDBRestReplicationHandler::processRestoreCollection( int RocksDBRestReplicationHandler::processRestoreCollectionCoordinator( VPackSlice const& collection, bool dropExisting, bool reuseId, bool force, uint64_t numberOfShards, std::string& errorMsg, - uint64_t replicationFactor) { + uint64_t replicationFactor, bool ignoreDistributeShardsLikeErrors) { if (!collection.isObject()) { errorMsg = "collection declaration is invalid"; @@ -2488,7 +2493,8 @@ int RocksDBRestReplicationHandler::processRestoreCollectionCoordinator( "Cluster") ->createWaitsForSyncReplication(); auto col = ClusterMethods::createCollectionOnCoordinator( - collectionType, _vocbase, merged, true, createWaitsForSyncReplication); + collectionType, _vocbase, merged, ignoreDistributeShardsLikeErrors, + createWaitsForSyncReplication); TRI_ASSERT(col != nullptr); } catch (basics::Exception const& e) { // Error, report it. diff --git a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.h b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.h index 786d84b438..930089b49c 100644 --- a/arangod/RocksDBEngine/RocksDBRestReplicationHandler.h +++ b/arangod/RocksDBEngine/RocksDBRestReplicationHandler.h @@ -260,7 +260,7 @@ class RocksDBRestReplicationHandler : public RestVocbaseBaseHandler { ////////////////////////////////////////////////////////////////////////////// int processRestoreCollectionCoordinator(VPackSlice const&, bool, bool, bool, - uint64_t, std::string&, uint64_t); + uint64_t, std::string&, uint64_t, bool); ////////////////////////////////////////////////////////////////////////////// /// @brief creates a collection, based on the VelocyPack provided TODO: MOVE