Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix handling of range deletions (#6639)

This commit is contained in:
Jan 2018-09-28 11:09:41 +02:00 committed by GitHub
parent 594e577ab0
commit 55cd434aa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1531 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
arangod/IResearch/IResearchRocksDBRecoveryHelper.cpp
View File

@ -422,6 +422,13 @@ void IResearchRocksDBRecoveryHelper::DeleteCF(uint32_t column_family_id,
void IResearchRocksDBRecoveryHelper::SingleDeleteCF(uint32_t column_family_id,
const rocksdb::Slice& key) {
// not needed for anything atm
}
void IResearchRocksDBRecoveryHelper::DeleteRangeCF(uint32_t column_family_id,
const rocksdb::Slice& end_key,
const rocksdb::Slice& begin_key) {
// not needed for anything atm
}
void IResearchRocksDBRecoveryHelper::LogData(const rocksdb::Slice& blob) {
@ -450,9 +457,20 @@ void IResearchRocksDBRecoveryHelper::LogData(const rocksdb::Slice& blob) {
TRI_idx_iid_t const indexId = RocksDBLogValue::indexId(blob);
dropCollectionFromView(*_dbFeature, dbId, collectionId, indexId, viewId);
} break;
default: {
// shut up the compiler
} break;
case RocksDBLogType::CollectionTruncate: {
uint64_t objectId = RocksDBLogValue::objectId(blob);
auto coll = lookupCollection(*_dbFeature, *_engine, objectId);
if (coll != nullptr) {
auto const links = lookupLinks(*coll);
for (auto link : links) {
link->afterTruncate();
}
}
break;
}
default: break; // shut up the compiler
}
}
@ -461,4 +479,4 @@ void IResearchRocksDBRecoveryHelper::LogData(const rocksdb::Slice& blob) {
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

4
arangod/IResearch/IResearchRocksDBRecoveryHelper.h
View File

@ -72,6 +72,10 @@ class IResearchRocksDBRecoveryHelper final : public RocksDBRecoveryHelper {
virtual void SingleDeleteCF(uint32_t column_family_id,
const rocksdb::Slice& key) override;
virtual void DeleteRangeCF(uint32_t column_family_id,
const rocksdb::Slice& begin_key,
const rocksdb::Slice& end_key) override;
virtual void LogData(const rocksdb::Slice& blob) override;

2
arangod/MMFiles/MMFilesCollection.cpp
View File

@ -2853,8 +2853,6 @@ Result MMFilesCollection::insert(transaction::Methods* trx,
OperationOptions& options,
TRI_voc_tick_t& resultMarkerTick, bool lock,
TRI_voc_tick_t& revisionId) {
VPackSlice fromSlice;
VPackSlice toSlice;
LocalDocumentId const documentId = reuseOrCreateLocalDocumentId(options);
auto isEdgeCollection = (TRI_COL_TYPE_EDGE == _logicalCollection.type());
transaction::BuilderLeaser builder(trx);

39
arangod/RocksDBEngine/RocksDBCollection.cpp
View File

@ -272,13 +272,14 @@ uint64_t RocksDBCollection::numberDocuments(transaction::Methods* trx) const {
/// @brief report extra memory used by indexes etc.
size_t RocksDBCollection::memory() const { return 0; }
void RocksDBCollection::open(bool ignoreErrors) {
void RocksDBCollection::open(bool /*ignoreErrors*/) {
TRI_ASSERT(_objectId != 0);
// set the initial number of documents
RocksDBEngine* engine =
static_cast<RocksDBEngine*>(EngineSelectorFeature::ENGINE);
auto counterValue = engine->settingsManager()->loadCounter(this->objectId());
TRI_ASSERT(engine != nullptr);
auto counterValue = engine->settingsManager()->loadCounter(_objectId);
_numberDocuments = counterValue.added() - counterValue.removed();
_revisionId = counterValue.revisionId();
}
@ -641,9 +642,8 @@ Result RocksDBCollection::truncate(transaction::Methods* trx,
// non-transactional truncate optimization. We perform a bunch of
// range deletes and circumwent the normal rocksdb::Transaction.
// no savepoint needed here
rocksdb::WriteBatch batch;
// add the assertion again here, so we are sure we can use RangeDeletes
TRI_ASSERT(static_cast<RocksDBEngine*>(EngineSelectorFeature::ENGINE)->canUseRangeDeleteInWal());
@ -675,19 +675,20 @@ Result RocksDBCollection::truncate(transaction::Methods* trx,
if (!s.ok()) {
return rocksutils::convertStatus(s);
}
idx->afterTruncate(); // clears caches (if applicable)
idx->afterTruncate(); // clears caches / clears links (if applicable)
}
}
state->addTruncateOperation(_logicalCollection.id());
rocksdb::WriteOptions wo;
s = rocksutils::globalRocksDB()->Write(wo, &batch);
if (!s.ok()) {
return rocksutils::convertStatus(s);
}
uint64_t prevCount = _numberDocuments;
_numberDocuments = 0; // protected by collection lock
TRI_ASSERT(state->numRemoves() == _numberDocuments);
if (prevCount > 64 * 1024) {
if (_numberDocuments > 64 * 1024) {
// also compact the ranges in order to speed up all further accesses
compact();
}
@ -1802,7 +1803,7 @@ uint64_t RocksDBCollection::recalculateCounts() {
rocksutils::countKeyRange(engine->db(), documentBounds, true);
// update counter manager value
res = engine->settingsManager()->setAbsoluteCounter(_objectId,
res = engine->settingsManager()->setAbsoluteCounter(_objectId, engine->currentTick(),
_numberDocuments);
if (res.ok()) {
// in case of fail the counter has never been written and hence does not
@ -1919,17 +1920,12 @@ void RocksDBCollection::recalculateIndexEstimates(
// issues with estimate integrity; please do not expose via a user-facing
// method or endpoint unless the implementation changes
// start transaction to get a collection lock
auto ctx =
transaction::StandaloneContext::Create(_logicalCollection.vocbase());
arangodb::SingleCollectionTransaction trx(
ctx, _logicalCollection, AccessMode::Type::EXCLUSIVE
);
auto res = trx.begin();
if (res.fail()) {
THROW_ARANGO_EXCEPTION(res);
}
// intentionally do not use transactions here, as we will only be called
// during recovery
RocksDBEngine* engine =
static_cast<RocksDBEngine*>(EngineSelectorFeature::ENGINE);
TRI_ASSERT(engine != nullptr);
TRI_ASSERT(engine->inRecovery());
for (auto const& it : indexes) {
auto idx = static_cast<RocksDBIndex*>(it.get());
@ -1937,8 +1933,6 @@ void RocksDBCollection::recalculateIndexEstimates(
TRI_ASSERT(idx != nullptr);
idx->recalculateEstimates();
}
trx.commit();
}
arangodb::Result RocksDBCollection::serializeKeyGenerator(
@ -1977,7 +1971,6 @@ void RocksDBCollection::deserializeKeyGenerator(RocksDBSettingsManager* mgr) {
if (value > 0) {
std::string k(basics::StringUtils::itoa(value));
_logicalCollection.keyGenerator()->track(k.data(), k.size());
}
}

2
arangod/RocksDBEngine/RocksDBEngine.cpp
View File

@ -1162,6 +1162,8 @@ bool RocksDBEngine::inRecovery() {
void RocksDBEngine::recoveryDone(TRI_vocbase_t& vocbase) {
// nothing to do here
settingsManager()->clearIndexEstimators();
settingsManager()->clearKeyGenerators();
}
std::string RocksDBEngine::createCollection(

4
arangod/RocksDBEngine/RocksDBRecoveryHelper.h
View File

@ -45,6 +45,10 @@ class RocksDBRecoveryHelper {
virtual void SingleDeleteCF(uint32_t column_family_id,
const rocksdb::Slice& key) {}
virtual void DeleteRangeCF(uint32_t column_family_id,
const rocksdb::Slice& begin_key,
const rocksdb::Slice& end_key) {}
virtual void LogData(const rocksdb::Slice& blob) {}
};

143
arangod/RocksDBEngine/RocksDBRecoveryManager.cpp
View File

@ -118,13 +118,25 @@ bool RocksDBRecoveryManager::inRecovery() const { return _inRecovery.load(); }
class WBReader final : public rocksdb::WriteBatch::Handler {
public:
std::unordered_map<uint64_t, RocksDBSettingsManager::CounterAdjustment>
deltas;
struct Operations {
explicit Operations(rocksdb::SequenceNumber seq) : startSequenceNumber(seq) {};
Operations(Operations const&) = delete;
Operations& operator=(Operations const&) = delete;
const rocksdb::SequenceNumber startSequenceNumber;
rocksdb::SequenceNumber lastSequenceNumber;
uint64_t added = 0;
uint64_t removed = 0;
TRI_voc_rid_t lastRevisionId = 0;
bool mustTruncate = false;
};
std::unordered_map<uint64_t, WBReader::Operations> deltas;
rocksdb::SequenceNumber currentSeqNum;
private:
// must be retrieved from settings manager
std::unordered_map<uint64_t, rocksdb::SequenceNumber> _seqStart;
// used to track used IDs for key-generators
std::unordered_map<uint64_t, uint64_t> _generators;
uint64_t _maxTick = 0;
@ -133,8 +145,16 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
TRI_voc_rid_t _lastRemovedDocRid = 0;
public:
/// @param seqs sequence number from which to count operations
explicit WBReader(std::unordered_map<uint64_t, rocksdb::SequenceNumber> const& seqs)
: currentSeqNum(0), _seqStart(seqs) {}
: currentSeqNum(0) {
for (auto const& pair : seqs) {
try {
deltas.emplace(pair.first, pair.second);
} catch(...) {}
}
}
Result shutdownWBReader() {
Result rv = basics::catchVoidToResult([&]() -> void {
@ -172,16 +192,15 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
return rv;
}
bool shouldHandleDocument(const rocksdb::Slice& key) {
uint64_t objectId = RocksDBKey::objectId(key);
auto const& it = _seqStart.find(objectId);
if (it != _seqStart.end()) {
if (deltas.find(objectId) == deltas.end()) {
deltas.emplace(objectId, RocksDBSettingsManager::CounterAdjustment());
}
return it->second <= currentSeqNum;
bool shouldHandleCollection(uint64_t objectId, Operations** ops) {
auto it = deltas.find(objectId);
if (it != deltas.end()) {
*ops = &(it->second);
return it->second.startSequenceNumber <= currentSeqNum;
}
return false;
auto res = deltas.emplace(objectId, currentSeqNum); // do not ignore unknown counters
*ops = &(res.first->second);
return true;
}
void storeMaxHLC(uint64_t hlc) {
@ -202,7 +221,6 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
}
auto it = _generators.find(objectId);
if (it == _generators.end()) {
try {
_generators.emplace(objectId, keyValue);
@ -304,15 +322,14 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
rocksdb::Status PutCF(uint32_t column_family_id, const rocksdb::Slice& key,
const rocksdb::Slice& value) override {
updateMaxTick(column_family_id, key, value);
if (column_family_id == RocksDBColumnFamily::documents()->GetID() &&
shouldHandleDocument(key)) {
if (column_family_id == RocksDBColumnFamily::documents()->GetID()) {
uint64_t objectId = RocksDBKey::objectId(key);
auto const& it = deltas.find(objectId);
if (it != deltas.end()) {
it->second._sequenceNum = currentSeqNum;
it->second._added++;
it->second._revisionId = transaction::helpers::extractRevFromDocument(RocksDBValue::data(value));
Operations* ops = nullptr;
if (shouldHandleCollection(objectId, &ops)) {
TRI_ASSERT(ops != nullptr);
ops->lastSequenceNumber = currentSeqNum;
ops->added++;
ops->lastRevisionId = transaction::helpers::extractRevFromDocument(RocksDBValue::data(value));
}
} else {
// We have to adjust the estimate with an insert
@ -345,18 +362,16 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
rocksdb::Status DeleteCF(uint32_t column_family_id,
const rocksdb::Slice& key) override {
if (column_family_id == RocksDBColumnFamily::documents()->GetID()) {
if (shouldHandleDocument(key)) {
uint64_t objectId = RocksDBKey::objectId(key);
auto const& it = deltas.find(objectId);
if (it != deltas.end()) {
it->second._sequenceNum = currentSeqNum;
it->second._removed++;
if (_lastRemovedDocRid != 0) {
it->second._revisionId = _lastRemovedDocRid;
}
uint64_t objectId = RocksDBKey::objectId(key);
Operations* ops = nullptr;
if (shouldHandleCollection(objectId, &ops)) {
TRI_ASSERT(ops != nullptr);
ops->lastSequenceNumber = currentSeqNum;
ops->removed++;
if (_lastRemovedDocRid != 0) {
ops->lastRevisionId = _lastRemovedDocRid;
}
}
}
_lastRemovedDocRid = 0; // reset in any case
} else {
// We have to adjust the estimate with an insert
@ -396,21 +411,24 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
return rocksdb::Status();
}
rocksdb::Status DeleteRangeCF(uint32_t column_family_id,
const rocksdb::Slice& begin_key,
const rocksdb::Slice& end_key) override {
// drop and truncate can use this, truncate is handled via a Log marker
RocksDBEngine* engine =
static_cast<RocksDBEngine*>(EngineSelectorFeature::ENGINE);
for (auto helper : engine->recoveryHelpers()) {
helper->DeleteRangeCF(column_family_id, begin_key, end_key);
}
return rocksdb::Status(); // make WAL iterator happy
}
void LogData(const rocksdb::Slice& blob) override {
// a delete log message appears directly before a Delete
RocksDBLogType type = RocksDBLogValue::type(blob);
switch(type) {
case RocksDBLogType::CollectionTruncate: {
uint64_t objectId = RocksDBLogValue::objectId(blob);
auto const& it = deltas.find(objectId);
if (it != deltas.end()) {
it->second._removed = 0;
it->second._added = 0;
}
_lastRemovedDocRid = 0; // reset in any other case
break;
}
case RocksDBLogType::DocumentRemoveV2: // remove within a trx
TRI_ASSERT(_lastRemovedDocRid == 0);
_lastRemovedDocRid = RocksDBLogValue::revisionId(blob);
@ -419,6 +437,25 @@ class WBReader final : public rocksdb::WriteBatch::Handler {
TRI_ASSERT(_lastRemovedDocRid == 0);
_lastRemovedDocRid = RocksDBLogValue::revisionId(blob);
break;
case RocksDBLogType::CollectionTruncate: {
uint64_t objectId = RocksDBLogValue::objectId(blob);
Operations* ops = nullptr;
if (shouldHandleCollection(objectId, &ops)) {
TRI_ASSERT(ops != nullptr);
ops->lastSequenceNumber = currentSeqNum;
ops->removed = 0;
ops->added = 0;
ops->mustTruncate = true;
auto est = findEstimator(objectId);
if (est != nullptr && est->commitSeq() < currentSeqNum) {
// We track estimates for this index
est->bufferTruncate(currentSeqNum + 1);
}
}
_lastRemovedDocRid = 0; // reset in any other case
break;
}
default:
_lastRemovedDocRid = 0; // reset in any other case
break;
@ -478,11 +515,23 @@ Result RocksDBRecoveryManager::parseRocksWAL() {
if (rv.ok()) {
LOG_TOPIC(TRACE, Logger::ENGINES)
<< "finished WAL scan with " << handler.deltas.size();
RocksDBSettingsManager* mgr = engine->settingsManager();
for (auto& pair : handler.deltas) {
engine->settingsManager()->updateCounter(pair.first, pair.second);
WBReader::Operations const& ops = pair.second;
if (ops.mustTruncate) {
mgr->setAbsoluteCounter(pair.first, ops.lastSequenceNumber, 0);
}
RocksDBSettingsManager::CounterAdjustment adj{};
adj._sequenceNum = ops.lastSequenceNumber;
adj._added = ops.added;
adj._removed = ops.removed;
adj._revisionId = ops.lastRevisionId;
mgr->updateCounter(pair.first, adj);
LOG_TOPIC(TRACE, Logger::ENGINES)
<< "WAL recovered " << pair.second.added() << " PUTs and "
<< pair.second.removed() << " DELETEs for objectID " << pair.first;
<< "WAL recovered " << adj.added() << " PUTs and "
<< adj.removed() << " DELETEs for objectID " << pair.first;
}
}
}

69
arangod/RocksDBEngine/RocksDBSettingsManager.cpp
View File

@ -203,7 +203,7 @@ writeIndexEstimatorsAndKeyGenerator(
namespace arangodb {
RocksDBSettingsManager::CMValue::CMValue(VPackSlice const& slice)
: _sequenceNum(0), _count(0), _revisionId(0) {
: _sequenceNum(0), _added(0), _removed(0), _revisionId(0) {
if (!slice.isArray()) {
// got a somewhat invalid slice. probably old data from before the key
// structure changes
@ -213,16 +213,23 @@ RocksDBSettingsManager::CMValue::CMValue(VPackSlice const& slice)
velocypack::ArrayIterator array(slice);
if (array.valid()) {
this->_sequenceNum = (*array).getUInt();
this->_count = (*(++array)).getUInt();
// versions pre 3.4 stored only a single "count" value
// 3.4 and higher store "added" and "removed" seperately
this->_added = (*(++array)).getUInt();
if (array.size() > 3) {
TRI_ASSERT(array.size() == 4);
this->_removed = (*(++array)).getUInt();
}
this->_revisionId = (*(++array)).getUInt();
}
}
void RocksDBSettingsManager::CMValue::serialize(VPackBuilder& b) const {
b.openArray();
b.add(VPackValue(this->_sequenceNum));
b.add(VPackValue(this->_count));
b.add(VPackValue(this->_revisionId));
b.add(VPackValue(_sequenceNum));
b.add(VPackValue(_added));
b.add(VPackValue(_removed));
b.add(VPackValue(_revisionId));
b.close();
}
@ -277,7 +284,9 @@ RocksDBSettingsManager::CounterAdjustment RocksDBSettingsManager::loadCounter(
auto const& it = _counters.find(objectId);
if (it != _counters.end()) {
return CounterAdjustment(it->second._sequenceNum, it->second._count, 0,
return CounterAdjustment(it->second._sequenceNum,
it->second._added,
it->second._removed,
it->second._revisionId);
}
@ -296,18 +305,20 @@ void RocksDBSettingsManager::updateCounter(uint64_t objectId,
auto it = _counters.find(objectId);
if (it != _counters.end()) {
it->second._count += update.added();
it->second._count -= update.removed();
it->second._added += update.added();
it->second._removed += update.removed();
// just use the latest trx info
if (seqNo > it->second._sequenceNum) {
it->second._sequenceNum = seqNo;
it->second._revisionId = update.revisionId();
if (update.revisionId() != 0) {
it->second._revisionId = update.revisionId();
}
}
} else {
// insert new counter
_counters.emplace(std::make_pair(
objectId,
CMValue(seqNo, update.added() - update.removed(),
CMValue(update.sequenceNumber(), update.added(), update.removed(),
update.revisionId())));
needsSync = true; // only count values from WAL if they are in the DB
}
@ -321,6 +332,7 @@ void RocksDBSettingsManager::updateCounter(uint64_t objectId,
}
arangodb::Result RocksDBSettingsManager::setAbsoluteCounter(uint64_t objectId,
rocksdb::SequenceNumber seq,
uint64_t value) {
arangodb::Result res;
rocksdb::SequenceNumber seqNo = 0;
@ -331,10 +343,9 @@ arangodb::Result RocksDBSettingsManager::setAbsoluteCounter(uint64_t objectId,
auto it = _counters.find(objectId);
if (it != _counters.end()) {
rocksdb::TransactionDB* db = rocksutils::globalRocksDB();
seqNo = db->GetLatestSequenceNumber();
it->second._sequenceNum = seqNo;
it->second._count = value;
it->second._sequenceNum = std::max(seq, it->second._sequenceNum);
it->second._added = value;
it->second._removed = 0;
} else {
// nothing to do as the counter has never been written it can not be set to
// a value that would require correction. but we use the return value to
@ -481,7 +492,6 @@ Result RocksDBSettingsManager::sync(bool force) {
}
void RocksDBSettingsManager::loadSettings() {
WRITE_LOCKER(guard, _rwLock);
RocksDBKey key;
key.constructSettingsValue(RocksDBSettingsType::ServerTick);
@ -497,6 +507,7 @@ void RocksDBSettingsManager::loadSettings() {
<< "read initial settings: " << slice.toJson();
if (!result.empty()) {
WRITE_LOCKER(guard, _rwLock);
try {
if (slice.hasKey("tick")) {
uint64_t lastTick =
@ -535,8 +546,6 @@ void RocksDBSettingsManager::loadSettings() {
}
void RocksDBSettingsManager::loadIndexEstimates() {
WRITE_LOCKER(guard, _rwLock);
RocksDBKeyBounds bounds = RocksDBKeyBounds::IndexEstimateValues();
auto cf = RocksDBColumnFamily::definitions();
@ -553,6 +562,8 @@ void RocksDBSettingsManager::loadIndexEstimates() {
StringRef estimateSerialization(iter->value().data() + sizeof(uint64_t),
iter->value().size() - sizeof(uint64_t));
WRITE_LOCKER(guard, _rwLock);
// If this hits we have two estimates for the same index
TRI_ASSERT(_estimators.find(objectId) == _estimators.end());
try {
@ -578,7 +589,6 @@ void RocksDBSettingsManager::loadIndexEstimates() {
}
void RocksDBSettingsManager::loadKeyGenerators() {
WRITE_LOCKER(guard, _rwLock);
RocksDBKeyBounds bounds = RocksDBKeyBounds::KeyGenerators();
auto cf = RocksDBColumnFamily::definitions();
@ -596,6 +606,7 @@ void RocksDBSettingsManager::loadKeyGenerators() {
if (!s.isNone()) {
uint64_t lastValue = properties.get(StaticStrings::LastValue).getUInt();
WRITE_LOCKER(guard, _rwLock);
// If this hits we have two generators for the same collection
TRI_ASSERT(_generators.find(objectId) == _generators.end());
try {
@ -647,8 +658,7 @@ uint64_t RocksDBSettingsManager::stealKeyGenerator(uint64_t objectId) {
void RocksDBSettingsManager::clearIndexEstimators() {
// We call this to remove all index estimators that have been stored but are
// no longer read
// by recovery.
// no longer read by recovery.
// TODO REMOVE RocksDB Keys of all not stolen values?
WRITE_LOCKER(guard, _rwLock);
@ -662,7 +672,6 @@ void RocksDBSettingsManager::clearKeyGenerators() {
/// Parse counter values from rocksdb
void RocksDBSettingsManager::loadCounterValues() {
WRITE_LOCKER(guard, _rwLock);
RocksDBKeyBounds bounds = RocksDBKeyBounds::CounterValues();
auto cf = RocksDBColumnFamily::definitions();
@ -673,13 +682,17 @@ void RocksDBSettingsManager::loadCounterValues() {
while (iter->Valid() && cmp->Compare(iter->key(), bounds.end()) < 0) {
uint64_t objectId = RocksDBKey::definitionsObjectId(iter->key());
auto const& it =
_counters.emplace(objectId, CMValue(VPackSlice(iter->value().data())));
_syncedSeqNums[objectId] = it.first->second._sequenceNum;
LOG_TOPIC(TRACE, Logger::ENGINES)
<< "found count marker for objectId '" << objectId
<< "' last synced at " << it.first->first << " with count "
<< it.first->second._count;
{
WRITE_LOCKER(guard, _rwLock);
auto const& it =
_counters.emplace(objectId, CMValue(VPackSlice(iter->value().data())));
_syncedSeqNums[objectId] = it.first->second._sequenceNum;
LOG_TOPIC(TRACE, Logger::ENGINES)
<< "found count marker for objectId '" << objectId
<< "' last synced at " << it.first->first << " with added "
<< it.first->second._added << ", removed " << it.first->second._removed;
}
iter->Next();
}

8
arangod/RocksDBEngine/RocksDBSettingsManager.h
View File

@ -77,12 +77,13 @@ class RocksDBSettingsManager {
/// ArangoDB transaction ID
rocksdb::SequenceNumber _sequenceNum;
/// used for number of documents
uint64_t _count;
uint64_t _added;
uint64_t _removed;
/// used for revision id
TRI_voc_rid_t _revisionId;
CMValue(rocksdb::SequenceNumber sq, uint64_t cc, TRI_voc_rid_t rid)
: _sequenceNum(sq), _count(cc), _revisionId(rid) {}
CMValue(rocksdb::SequenceNumber sq, uint64_t added, uint64_t removed, TRI_voc_rid_t rid)
: _sequenceNum(sq), _added(added), _removed(removed), _revisionId(rid) {}
explicit CMValue(arangodb::velocypack::Slice const&);
void serialize(arangodb::velocypack::Builder&) const;
};
@ -101,6 +102,7 @@ class RocksDBSettingsManager {
// does not modify seq or revisionid
arangodb::Result setAbsoluteCounter(uint64_t objectId,
rocksdb::SequenceNumber,
uint64_t absouluteCount);
/// Thread-Safe remove a counter

11
arangod/RocksDBEngine/RocksDBTransactionCollection.cpp
View File

@ -111,7 +111,7 @@ bool RocksDBTransactionCollection::isLocked() const {
if (_collection == nullptr) {
return false;
}
std::string collName(_collection->name());
std::string collName(_collection->name()); // WTF?!
if (_transaction->isLockedShard(collName)) {
return true;
}
@ -277,6 +277,15 @@ void RocksDBTransactionCollection::addOperation(
}
}
void RocksDBTransactionCollection::addTruncateOperation() {
TRI_ASSERT(_numInserts == 0 && _numUpdates == 0 && _numRemoves == 0);
if (!isLocked() || _accessType != AccessMode::Type::EXCLUSIVE) {
TRI_ASSERT(false);
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "collection must be exlusively locked");
}
_numRemoves += _initialNumberDocuments + _numInserts;
}
void RocksDBTransactionCollection::prepareCommit(uint64_t trxId,
uint64_t preCommitSeq) {
TRI_ASSERT(_collection != nullptr);

5
arangod/RocksDBEngine/RocksDBTransactionCollection.h
View File

@ -88,6 +88,11 @@ class RocksDBTransactionCollection final : public TransactionCollection {
/// @brief add an operation for a transaction collection
void addOperation(TRI_voc_document_operation_e operationType,
TRI_voc_rid_t revisionId);
/// @brief will perform _numRemoves = _initialNumberDocuments
/// be aware that this is only a valid operation under an
/// exclusive collection lock
void addTruncateOperation();
/**
* @brief Prepare collection for commit by placing index blockers

35
arangod/RocksDBEngine/RocksDBTransactionState.cpp
View File

@ -269,8 +269,8 @@ arangodb::Result RocksDBTransactionState::internalCommit() {
}
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
// sanity check for our on-disk WAL format
uint64_t x = _numInserts + _numRemoves + _numUpdates;
if (hasHint(transaction::Hints::Hint::SINGLE_OPERATION)) {
TRI_ASSERT(x <= 1 && _numLogdata == x);
} else {
@ -280,7 +280,6 @@ arangodb::Result RocksDBTransactionState::internalCommit() {
<< "_numUpdates " << _numUpdates << " "
<< "_numLogdata " << _numLogdata;
}
// begin transaction + commit transaction + n doc removes
TRI_ASSERT(_numLogdata == (2 + _numRemoves));
}
@ -318,10 +317,11 @@ arangodb::Result RocksDBTransactionState::internalCommit() {
++_numCommits;
result = rocksutils::convertStatus(_rocksTransaction->Commit());
rocksdb::SequenceNumber latestSeq =
rocksutils::globalRocksDB()->GetLatestSequenceNumber();
if (result.ok()) {
rocksdb::SequenceNumber latestSeq =
rocksutils::globalRocksDB()->GetLatestSequenceNumber();
for (auto& trxCollection : _collections) {
RocksDBTransactionCollection* collection =
static_cast<RocksDBTransactionCollection*>(trxCollection);
@ -515,22 +515,20 @@ Result RocksDBTransactionState::addOperation(
return Result(Result(TRI_ERROR_RESOURCE_LIMIT, message));
}
auto collection =
static_cast<RocksDBTransactionCollection*>(findCollection(cid));
if (collection == nullptr) {
auto tcoll = static_cast<RocksDBTransactionCollection*>(findCollection(cid));
if (tcoll == nullptr) {
std::string message = "collection '" + std::to_string(cid) +
"' not found in transaction state";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, message);
}
// should not fail or fail with exception
collection->addOperation(operationType, revisionId);
tcoll->addOperation(operationType, revisionId);
// clear the query cache for this collection
auto queryCache = arangodb::aql::QueryCache::instance();
if (queryCache->mayBeActive()) {
queryCache->invalidate(&_vocbase, collection->collectionName());
queryCache->invalidate(&_vocbase, tcoll->collectionName());
}
switch (operationType) {
@ -552,6 +550,23 @@ Result RocksDBTransactionState::addOperation(
return checkIntermediateCommit(currentSize, hasPerformedIntermediateCommit);
}
// only a valid under an exlusive lock as an only operation
void RocksDBTransactionState::addTruncateOperation(TRI_voc_cid_t cid) {
auto tcoll = static_cast<RocksDBTransactionCollection*>(findCollection(cid));
if (tcoll == nullptr) {
std::string message = "collection '" + std::to_string(cid) +
"' not found in transaction state";
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, message);
}
tcoll->addTruncateOperation();
_numRemoves += tcoll->numRemoves();
TRI_ASSERT(_numInserts == 0 && _numUpdates == 0);
TRI_ASSERT(!hasHint(transaction::Hints::Hint::SINGLE_OPERATION));
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
_numLogdata += _numRemoves; // cheat our own sanity checks
#endif
}
RocksDBMethods* RocksDBTransactionState::rocksdbMethods() {
TRI_ASSERT(_rocksMethods);
return _rocksMethods.get();

5
arangod/RocksDBEngine/RocksDBTransactionState.h
View File

@ -116,6 +116,11 @@ class RocksDBTransactionState final : public TransactionState {
Result addOperation(TRI_voc_cid_t collectionId,
TRI_voc_rid_t revisionId, TRI_voc_document_operation_e opType,
bool& hasPerformedIntermediateCommit);
/// @brief will perform _numRemoves = _initialNumberDocuments
/// be aware that this is only a valid operation under an
/// exclusive collection lock
void addTruncateOperation(TRI_voc_cid_t cid);
RocksDBMethods* rocksdbMethods();

5
arangod/RocksDBEngine/RocksDBVPackIndex.cpp
View File

@ -1223,7 +1223,7 @@ bool RocksDBVPackIndex::deserializeEstimate(RocksDBSettingsManager* mgr) {
return true;
}
// We simply drop the current estimator and steal the one from recovery
// We are than save for resizing issues in our _estimator format
// We are then safe for resizing issues in our _estimator format
// and will use the old size.
TRI_ASSERT(mgr != nullptr);
@ -1256,6 +1256,9 @@ void RocksDBVPackIndex::recalculateEstimates() {
}
void RocksDBVPackIndex::afterTruncate() {
if (unique()) {
return;
}
TRI_ASSERT(_estimator != nullptr);
_estimator->bufferTruncate(rocksutils::latestSequenceNumber());
RocksDBIndex::afterTruncate();

97
tests/js/server/recovery/nosync-rangedelete-collection-drop-indexes-rocksdb.js Normal file
View File

@ -0,0 +1,97 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
// write some documents with autoincrement keys
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
db._drop('UnitTestsRecovery1');
c = db._create('UnitTestsRecovery2');
c.save({ }, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNoSyncRangeDeleteCollectionDropIndexes: function () {
assertNull(db._collection('UnitTestsRecovery1'));
assertNotNull(db._collection('UnitTestsRecovery2'));
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

94
tests/js/server/recovery/nosync-rangedelete-collection-drop-rocksdb.js Normal file
View File

@ -0,0 +1,94 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
// write some documents with autoincrement keys
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
db._drop('UnitTestsRecovery1');
c = db._create('UnitTestsRecovery2');
c.save({ }, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteCollectionDrop: function () {
assertNull(db._collection('UnitTestsRecovery1'));
assertNotNull(db._collection('UnitTestsRecovery2'));
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

106
tests/js/server/recovery/nosync-rangedelete-truncate-indexes-rocksdb.js Normal file
View File

@ -0,0 +1,106 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _key: "test" + i, _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
c.truncate();
c = db._create('UnitTestsRecovery2');
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteTruncateIndexes: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(0, c.count());
assertNotNull(db._collection('UnitTestsRecovery2'));
assertEqual([], c.edges("test/1"));
let query = "FOR doc IN @@collection FILTER doc.value == @value RETURN doc";
for (let i = 0; i < 100000; i += 1000) {
assertFalse(c.exists("key" + i));
assertEqual([], db._query(query, { "@collection": c.name(), value: i }).toArray());
assertEqual([], c.edges("test/" + i));
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

106
tests/js/server/recovery/nosync-rangedelete-truncate-multi-rocksdb.js Normal file
View File

@ -0,0 +1,106 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _key: "test" + i, _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
c.truncate();
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
c = db._create('UnitTestsRecovery2');
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteTruncateMulti: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(0, c.count());
assertNotNull(db._collection('UnitTestsRecovery2'));
assertEqual([], c.edges("test/1"));
let query = "FOR doc IN @@collection FILTER doc.value == @value RETURN doc";
for (let i = 0; i < 100000; i += 1000) {
assertFalse(c.exists("key" + i));
assertEqual([], db._query(query, { "@collection": c.name(), value: i }).toArray());
assertEqual([], c.edges("test/" + i));
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

101
tests/js/server/recovery/nosync-rangedelete-truncate-multi2-rocksdb.js Normal file
View File

@ -0,0 +1,101 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
c.truncate();
for (let i = 0; i < 90000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.truncate();
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteTruncate2: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(1, c.count());
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

100
tests/js/server/recovery/nosync-rangedelete-truncate-multi3-rocksdb.js Normal file
View File

@ -0,0 +1,100 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
c.truncate();
for (let i = 0; i < 90000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.truncate();
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteTruncate3: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(1, c.count());
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

99
tests/js/server/recovery/nosync-rangedelete-truncate-refill-rocksdb.js Normal file
View File

@ -0,0 +1,99 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// turn off syncing of counters etc.
internal.debugSetFailAt("RocksDBSettingsManagerSync");
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
c.truncate();
for (let i = 0; i < 5000; i++) {
docs.push({ _key: "test" + i, value: i });
}
c.insert(docs, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testNosyncRangeDeleteTruncateRefill: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(5000, c.count());
for (let i = 0; i < 5000; ++i) {
assertEqual(i, c.document("test" + i).value);
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

94
tests/js/server/recovery/rangedelete-collection-drop-indexes-rocksdb.js Normal file
View File

@ -0,0 +1,94 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// write some documents with autoincrement keys
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
db._drop('UnitTestsRecovery1');
c = db._create('UnitTestsRecovery2');
c.save({ }, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteCollectionDropIndexes: function () {
assertNull(db._collection('UnitTestsRecovery1'));
assertNotNull(db._collection('UnitTestsRecovery2'));
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

91
tests/js/server/recovery/rangedelete-collection-drop-rocksdb.js Normal file
View File

@ -0,0 +1,91 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
// write some documents with autoincrement keys
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
db._drop('UnitTestsRecovery1');
c = db._create('UnitTestsRecovery2');
c.save({ }, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteCollectionDrop: function () {
assertNull(db._collection('UnitTestsRecovery1'));
assertNotNull(db._collection('UnitTestsRecovery2'));
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

103
tests/js/server/recovery/rangedelete-truncate-indexes-rocksdb.js Normal file
View File

@ -0,0 +1,103 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _key: "test" + i, _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
c.truncate();
c = db._create('UnitTestsRecovery2');
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteTruncateIndexes: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(0, c.count());
assertNotNull(db._collection('UnitTestsRecovery2'));
assertEqual([], c.edges("test/1"));
let query = "FOR doc IN @@collection FILTER doc.value == @value RETURN doc";
for (let i = 0; i < 100000; i += 1000) {
assertFalse(c.exists("key" + i));
assertEqual([], db._query(query, { "@collection": c.name(), value: i }).toArray());
assertEqual([], c.edges("test/" + i));
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

103
tests/js/server/recovery/rangedelete-truncate-multi-rocksdb.js Normal file
View File

@ -0,0 +1,103 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
db._drop('UnitTestsRecovery1');
let c = db._createEdgeCollection('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ _key: "test" + i, _from: "test/1", _to: "test/" + i, value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.ensureIndex({ type: "hash", fields: ["value"] });
c.ensureIndex({ type: "hash", fields: ["value", "_to"], unique: true });
// should trigger range deletion
c.truncate();
c = db._create('UnitTestsRecovery2');
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteTruncateMulti: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(0, c.count());
assertNotNull(db._collection('UnitTestsRecovery2'));
assertEqual([], c.edges("test/1"));
let query = "FOR doc IN @@collection FILTER doc.value == @value RETURN doc";
for (let i = 0; i < 100000; i += 1000) {
assertFalse(c.exists("key" + i));
assertEqual([], db._query(query, { "@collection": c.name(), value: i }).toArray());
assertEqual([], c.edges("test/" + i));
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

98
tests/js/server/recovery/rangedelete-truncate-multi2-rocksdb.js Normal file
View File

@ -0,0 +1,98 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
c.truncate();
for (let i = 0; i < 90000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
c.truncate();
c.insert({}, { waitForSync: true });
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteTruncateMulti2: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(1, c.count());
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}

100
tests/js/server/recovery/rangedelete-truncate-refill-rocksdb.js Normal file
View File

@ -0,0 +1,100 @@
/* jshint globalstrict:false, strict:false, unused: false */
/* global assertEqual, assertFalse, assertNull, assertNotNull */
// //////////////////////////////////////////////////////////////////////////////
// / @brief tests for transactions
// /
// / @file
// /
// / DISCLAIMER
// /
// / Copyright 2010-2012 triagens GmbH, Cologne, Germany
// /
// / Licensed under the Apache License, Version 2.0 (the "License")
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / http://www.apache.org/licenses/LICENSE-2.0
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is triAGENS GmbH, Cologne, Germany
// /
// / @author Jan Steemann
// / @author Copyright 2013, triAGENS GmbH, Cologne, Germany
// //////////////////////////////////////////////////////////////////////////////
var db = require('@arangodb').db;
var internal = require('internal');
var jsunity = require('jsunity');
function runSetup () {
'use strict';
db._drop('UnitTestsRecovery1');
let c = db._create('UnitTestsRecovery1');
let docs = [];
for (let i = 0; i < 100000; i++) {
docs.push({ value: i });
if (docs.length === 10000) {
c.insert(docs);
docs = [];
}
}
// should trigger range deletion
c.truncate();
for (let i = 0; i < 5000; i++) {
docs.push({ _key: "test" + i, value: i });
}
c.insert(docs, { waitForSync: true });
// this should trigger the background sync thread at least once,
// though this is not guaranteed under high load
c.recalculateCounts();
internal.debugSegfault('crashing server');
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief test suite
// //////////////////////////////////////////////////////////////////////////////
function recoverySuite () {
'use strict';
jsunity.jsUnity.attachAssertions();
return {
setUp: function () {},
tearDown: function () {},
testRangeDeleteTruncateRefill: function () {
let c = db._collection('UnitTestsRecovery1');
assertEqual(5000, c.count());
for (let i = 0; i < 5000; ++i) {
assertEqual(i, c.document("test" + i).value);
}
}
};
}
// //////////////////////////////////////////////////////////////////////////////
// / @brief executes the test suite
// //////////////////////////////////////////////////////////////////////////////
function main (argv) {
'use strict';
if (argv[1] === 'setup') {
runSetup();
return 0;
} else {
jsunity.run(recoverySuite);
return jsunity.done().status ? 0 : 1;
}
}