Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

snafu (#8728)

This commit is contained in:
Simon 2019-04-10 19:14:34 +02:00 committed by Jan
parent 56336e74f8
commit 676012ca3a
9 changed files with 40 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arangod/RocksDBEngine/RocksDBCollection.cpp
View File

@ -657,7 +657,6 @@ Result RocksDBCollection::truncate(transaction::Methods& trx, OperationOptions&
seq = db->GetLatestSequenceNumber() - 1; // post commit sequence seq = db->GetLatestSequenceNumber() - 1; // post commit sequence
uint64_t numDocs = _numberDocuments.exchange(0); uint64_t numDocs = _numberDocuments.exchange(0);
_meta.adjustNumberDocuments(seq, /*revision*/ newRevisionId(), _meta.adjustNumberDocuments(seq, /*revision*/ newRevisionId(),
-static_cast<int64_t>(numDocs)); -static_cast<int64_t>(numDocs));

22
arangod/RocksDBEngine/RocksDBCollectionMeta.cpp
View File

@ -124,14 +124,14 @@ void RocksDBCollectionMeta::removeBlocker(TRI_voc_tid_t trxId) {
} }
/// @brief returns the largest safe seq to squash updates against /// @brief returns the largest safe seq to squash updates against
rocksdb::SequenceNumber RocksDBCollectionMeta::committableSeq() const { rocksdb::SequenceNumber RocksDBCollectionMeta::committableSeq(rocksdb::SequenceNumber maxCommitSeq) const {
READ_LOCKER(locker, _blockerLock); READ_LOCKER(locker, _blockerLock);
// if we have a blocker use the lowest counter // if we have a blocker use the lowest counter
if (!_blockersBySeq.empty()) { if (!_blockersBySeq.empty()) {
auto it = _blockersBySeq.begin(); auto it = _blockersBySeq.begin();
return it->first; return std::min(it->first, maxCommitSeq);
} }
return std::numeric_limits<rocksdb::SequenceNumber>::max(); return maxCommitSeq;
} }
rocksdb::SequenceNumber RocksDBCollectionMeta::applyAdjustments(rocksdb::SequenceNumber commitSeq, rocksdb::SequenceNumber RocksDBCollectionMeta::applyAdjustments(rocksdb::SequenceNumber commitSeq,
@ -172,8 +172,9 @@ rocksdb::SequenceNumber RocksDBCollectionMeta::applyAdjustments(rocksdb::Sequenc
/// @brief get the current count /// @brief get the current count
RocksDBCollectionMeta::DocCount RocksDBCollectionMeta::loadCount() { RocksDBCollectionMeta::DocCount RocksDBCollectionMeta::loadCount() {
auto maxxSeq = std::numeric_limits<rocksdb::SequenceNumber>::max();
bool didWork = false; bool didWork = false;
const rocksdb::SequenceNumber commitSeq = committableSeq(); const rocksdb::SequenceNumber commitSeq = committableSeq(maxxSeq);
applyAdjustments(commitSeq, didWork); applyAdjustments(commitSeq, didWork);
return _count; return _count;
} }
@ -191,14 +192,23 @@ Result RocksDBCollectionMeta::serializeMeta(rocksdb::WriteBatch& batch,
LogicalCollection& coll, bool force, LogicalCollection& coll, bool force,
VPackBuilder& tmp, VPackBuilder& tmp,
rocksdb::SequenceNumber& appliedSeq) { rocksdb::SequenceNumber& appliedSeq) {
TRI_ASSERT(appliedSeq != UINT64_MAX);
Result res; Result res;
bool didWork = false; bool didWork = false;
// maxCommitSeq is == UINT64_MAX without any blockers // maxCommitSeq is == UINT64_MAX without any blockers
const rocksdb::SequenceNumber maxCommitSeq = std::min(appliedSeq, committableSeq()); const rocksdb::SequenceNumber maxCommitSeq = committableSeq(appliedSeq);
const rocksdb::SequenceNumber commitSeq = applyAdjustments(maxCommitSeq, didWork); const rocksdb::SequenceNumber commitSeq = applyAdjustments(maxCommitSeq, didWork);
TRI_ASSERT(commitSeq <= appliedSeq); TRI_ASSERT(commitSeq <= appliedSeq);
appliedSeq = commitSeq; TRI_ASSERT(commitSeq <= maxCommitSeq);
TRI_ASSERT(maxCommitSeq <= appliedSeq);
TRI_ASSERT(maxCommitSeq != UINT64_MAX);
if (didWork) {
appliedSeq = commitSeq;
} else {
appliedSeq = maxCommitSeq;
}
RocksDBKey key; RocksDBKey key;
rocksdb::ColumnFamilyHandle* const cf = RocksDBColumnFamily::definitions(); rocksdb::ColumnFamilyHandle* const cf = RocksDBColumnFamily::definitions();

2
arangod/RocksDBEngine/RocksDBCollectionMeta.h
View File

@ -94,7 +94,7 @@ struct RocksDBCollectionMeta final {
void removeBlocker(TRI_voc_tid_t trxId); void removeBlocker(TRI_voc_tid_t trxId);
/// @brief returns the largest safe seq to squash updates against /// @brief returns the largest safe seq to squash updates against
rocksdb::SequenceNumber committableSeq() const; rocksdb::SequenceNumber committableSeq(rocksdb::SequenceNumber maxCommitSeq) const;
/// @brief get the current count /// @brief get the current count
DocCount loadCount(); DocCount loadCount();

1
arangod/RocksDBEngine/RocksDBEdgeIndex.cpp
View File

@ -1105,6 +1105,7 @@ RocksDBCuckooIndexEstimator<uint64_t>* RocksDBEdgeIndex::estimator() {
} }
void RocksDBEdgeIndex::setEstimator(std::unique_ptr<RocksDBCuckooIndexEstimator<uint64_t>> est) { void RocksDBEdgeIndex::setEstimator(std::unique_ptr<RocksDBCuckooIndexEstimator<uint64_t>> est) {
TRI_ASSERT(_estimator == nullptr || _estimator->appliedSeq() <= est->appliedSeq());
_estimator = std::move(est); _estimator = std::move(est);
} }

17
arangod/RocksDBEngine/RocksDBRecoveryManager.cpp
View File

@ -423,14 +423,15 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
cc._committedSeq = _currentSequence; cc._committedSeq = _currentSequence;
cc._added = 0; cc._added = 0;
cc._removed = 0; cc._removed = 0;
}
for (std::shared_ptr<arangodb::Index> const& idx : coll->getIndexes()) {
for (std::shared_ptr<arangodb::Index> const& idx : coll->getIndexes()) { RocksDBIndex* ridx = static_cast<RocksDBIndex*>(idx.get());
RocksDBIndex* ridx = static_cast<RocksDBIndex*>(idx.get()); RocksDBCuckooIndexEstimator<uint64_t>* est = ridx->estimator();
RocksDBCuckooIndexEstimator<uint64_t>* est = ridx->estimator(); TRI_ASSERT(ridx->type() != Index::TRI_IDX_TYPE_EDGE_INDEX || est);
if (est && est->appliedSeq() <= _currentSequence) { if (est) {
est->clear(); est->clear();
est->setAppliedSeq(_currentSequence); est->setAppliedSeq(_currentSequence);
}
} }
} }
} }

9
arangod/RocksDBEngine/RocksDBSettingsManager.cpp
View File

@ -202,14 +202,14 @@ Result RocksDBSettingsManager::sync(bool force) {
batch.Clear(); batch.Clear();
} }
TRI_ASSERT(_lastSync <= minSeqNr);
if (!didWork) { if (!didWork) {
WRITE_LOCKER(guard, _rwLock);
_lastSync = minSeqNr; _lastSync = minSeqNr;
return Result(); // nothing was written return Result(); // nothing was written
} }
_tmpBuilder.clear(); _tmpBuilder.clear();
Result res = writeSettings(batch, _tmpBuilder, std::max(_lastSync, minSeqNr)); Result res = writeSettings(batch, _tmpBuilder, std::max(_lastSync.load(), minSeqNr));
if (res.fail()) { if (res.fail()) {
LOG_TOPIC("8a5e6", WARN, Logger::ENGINES) LOG_TOPIC("8a5e6", WARN, Logger::ENGINES)
<< "could not store metadata settings " << res.errorMessage(); << "could not store metadata settings " << res.errorMessage();
@ -219,8 +219,7 @@ Result RocksDBSettingsManager::sync(bool force) {
// we have to commit all counters in one batch // we have to commit all counters in one batch
auto s = _db->Write(wo, &batch); auto s = _db->Write(wo, &batch);
if (s.ok()) { if (s.ok()) {
WRITE_LOCKER(guard, _rwLock); _lastSync = std::max(_lastSync.load(), minSeqNr);
_lastSync = std::max(_lastSync, minSeqNr);
} }
return rocksutils::convertStatus(s); return rocksutils::convertStatus(s);
@ -241,7 +240,6 @@ void RocksDBSettingsManager::loadSettings() {
LOG_TOPIC("7458b", TRACE, Logger::ENGINES) << "read initial settings: " << slice.toJson(); LOG_TOPIC("7458b", TRACE, Logger::ENGINES) << "read initial settings: " << slice.toJson();
if (!result.empty()) { if (!result.empty()) {
WRITE_LOCKER(guard, _rwLock);
try { try {
if (slice.hasKey("tick")) { if (slice.hasKey("tick")) {
uint64_t lastTick = uint64_t lastTick =
@ -279,7 +277,6 @@ void RocksDBSettingsManager::loadSettings() {
/// earliest safe sequence number to throw away from wal /// earliest safe sequence number to throw away from wal
rocksdb::SequenceNumber RocksDBSettingsManager::earliestSeqNeeded() const { rocksdb::SequenceNumber RocksDBSettingsManager::earliestSeqNeeded() const {
READ_LOCKER(guard, _rwLock);
return _lastSync; return _lastSync;
} }

5
arangod/RocksDBEngine/RocksDBSettingsManager.h
View File

@ -68,14 +68,11 @@ class RocksDBSettingsManager {
bool lockForSync(bool force); bool lockForSync(bool force);
private: private:
/// @brief protect _syncing and _counters
mutable basics::ReadWriteLock _rwLock;
/// @brief a reusable builder, used inside sync() to serialize objects /// @brief a reusable builder, used inside sync() to serialize objects
arangodb::velocypack::Builder _tmpBuilder; arangodb::velocypack::Builder _tmpBuilder;
/// @brief last sync sequence number /// @brief last sync sequence number
rocksdb::SequenceNumber _lastSync; std::atomic<rocksdb::SequenceNumber> _lastSync;
/// @brief currently syncing /// @brief currently syncing
std::atomic<bool> _syncing; std::atomic<bool> _syncing;

1
arangod/RocksDBEngine/RocksDBVPackIndex.cpp
View File

@ -1274,6 +1274,7 @@ RocksDBCuckooIndexEstimator<uint64_t>* RocksDBVPackIndex::estimator() {
void RocksDBVPackIndex::setEstimator(std::unique_ptr<RocksDBCuckooIndexEstimator<uint64_t>> est) { void RocksDBVPackIndex::setEstimator(std::unique_ptr<RocksDBCuckooIndexEstimator<uint64_t>> est) {
TRI_ASSERT(!_unique); TRI_ASSERT(!_unique);
TRI_ASSERT(_estimator == nullptr || _estimator->appliedSeq() <= est->appliedSeq());
_estimator = std::move(est); _estimator = std::move(est);
} }

16
tests/RocksDBEngine/IndexEstimatorTest.cpp
View File

@ -157,14 +157,14 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
est.bufferUpdates(++currentSeq, std::move(toInsert), std::move(toRemove)); est.bufferUpdates(++currentSeq, std::move(toInsert), std::move(toRemove));
// make sure we don't apply yet // make sure we don't apply yet
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
serialization.clear(); serialization.clear();
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);
REQUIRE((1.0 / std::max(1.0, static_cast<double>(iteration))) == REQUIRE((1.0 / std::max(1.0, static_cast<double>(iteration))) ==
est.computeEstimate()); est.computeEstimate());
meta.removeBlocker(iteration); meta.removeBlocker(iteration);
CHECK(meta.committableSeq() == UINT64_MAX); CHECK(meta.committableSeq(UINT64_MAX) == UINT64_MAX);
// now make sure we apply it // now make sure we apply it
est.serialize(serialization, currentSeq); est.serialize(serialization, currentSeq);
@ -188,8 +188,8 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
est.bufferUpdates(++currentSeq, std::move(toInsert), std::move(toRemove)); est.bufferUpdates(++currentSeq, std::move(toInsert), std::move(toRemove));
// make sure we don't apply yet // make sure we don't apply yet
REQUIRE(meta.committableSeq() == expected + 1); REQUIRE(meta.committableSeq(UINT64_MAX) == expected + 1);
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
serialization.clear(); serialization.clear();
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);
REQUIRE((1.0 / std::max(1.0, static_cast<double>(10 - iteration))) == REQUIRE((1.0 / std::max(1.0, static_cast<double>(10 - iteration))) ==
@ -198,7 +198,7 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
meta.removeBlocker(iteration); meta.removeBlocker(iteration);
// now make sure we apply it // now make sure we apply it
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
serialization.clear(); serialization.clear();
expected = currentSeq; expected = currentSeq;
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);
@ -232,7 +232,7 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
meta.removeBlocker(iteration - 1); meta.removeBlocker(iteration - 1);
// now make sure we applied last batch, but not this one // now make sure we applied last batch, but not this one
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
serialization.clear(); serialization.clear();
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);
REQUIRE((1.0 / std::max(1.0, static_cast<double>(iteration))) == REQUIRE((1.0 / std::max(1.0, static_cast<double>(iteration))) ==
@ -264,7 +264,7 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
meta.removeBlocker(std::max(static_cast<size_t>(1), iteration)); meta.removeBlocker(std::max(static_cast<size_t>(1), iteration));
// now make sure we haven't applied anything // now make sure we haven't applied anything
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
serialization.clear(); serialization.clear();
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);
REQUIRE(1.0 == est.computeEstimate()); REQUIRE(1.0 == est.computeEstimate());
@ -272,7 +272,7 @@ TEST_CASE("IndexEstimator", "[rocksdb][indexestimator]") {
// now remove first blocker and make sure we apply everything // now remove first blocker and make sure we apply everything
meta.removeBlocker(0); meta.removeBlocker(0);
est.serialize(serialization, meta.committableSeq()); est.serialize(serialization, meta.committableSeq(UINT64_MAX));
expected = currentSeq; expected = currentSeq;
serialization.clear(); serialization.clear();
REQUIRE(est.appliedSeq() == expected); REQUIRE(est.appliedSeq() == expected);