1
0
Fork 0

initial work for better incremental synchronization

This commit is contained in:
Jan Steemann 2015-09-10 18:17:58 +02:00
parent 03ae917d94
commit bead727a32
11 changed files with 1085 additions and 3 deletions

View File

@ -194,6 +194,8 @@ add_executable(
Scheduler/TimerTask.cpp
Statistics/statistics.cpp
Utils/CollectionExport.cpp
Utils/CollectionKeys.cpp
Utils/CollectionKeysRepository.cpp
Utils/Cursor.cpp
Utils/CursorRepository.cpp
Utils/DocumentHelper.cpp

View File

@ -152,6 +152,8 @@ arangod_libarangod_a_SOURCES = \
arangod/Scheduler/TimerTask.cpp \
arangod/Statistics/statistics.cpp \
arangod/Utils/CollectionExport.cpp \
arangod/Utils/CollectionKeys.cpp \
arangod/Utils/CollectionKeysRepository.cpp \
arangod/Utils/Cursor.cpp \
arangod/Utils/CursorRepository.cpp \
arangod/Utils/DocumentHelper.cpp \

View File

@ -177,7 +177,7 @@ bool RestQueryCacheHandler::readProperties () {
/// @startDocuBlock PutApiQueryCacheProperties
/// @brief changes the configuration for the AQL query cache
///
/// @RESTHEADER{PUT /_api/query-cache/properties, Changes the global properties for the AQL query cache}
/// @RESTHEADER{PUT /_api/query-cache/properties, Globally adjusts the AQL query result cache properties}
///
/// @RESTBODYPARAM{properties,json,required}
/// The global properties for AQL query cache.

View File

@ -42,6 +42,8 @@
#include "Replication/InitialSyncer.h"
#include "Rest/HttpRequest.h"
#include "Utils/CollectionGuard.h"
#include "Utils/CollectionKeys.h"
#include "Utils/CollectionKeysRepository.h"
#include "Utils/transactions.h"
#include "VocBase/compactor.h"
#include "VocBase/replication-applier.h"
@ -161,6 +163,27 @@ HttpHandler::status_t RestReplicationHandler::execute () {
handleCommandInventory();
}
}
else if (command == "keys") {
if (type != HttpRequest::HTTP_REQUEST_GET &&
type != HttpRequest::HTTP_REQUEST_POST &&
type != HttpRequest::HTTP_REQUEST_DELETE) {
goto BAD_CALL;
}
if (isCoordinatorError()) {
return status_t(HttpHandler::HANDLER_DONE);
}
if (type == HttpRequest::HTTP_REQUEST_POST) {
handleCommandCreateKeys();
}
else if (type == HttpRequest::HTTP_REQUEST_GET) {
handleCommandGetKeys();
}
else if (type == HttpRequest::HTTP_REQUEST_DELETE) {
handleCommandRemoveKeys();
}
}
else if (command == "dump") {
if (type != HttpRequest::HTTP_REQUEST_GET) {
goto BAD_CALL;
@ -1726,7 +1749,7 @@ void RestReplicationHandler::handleCommandClusterInventory () {
// Wrap the result:
TRI_json_t wrap;
TRI_InitObjectJson(TRI_CORE_MEM_ZONE, &wrap, 3);
TRI_Insert2ObjectJson(TRI_CORE_MEM_ZONE,&wrap,"collections", &json);
TRI_Insert2ObjectJson(TRI_CORE_MEM_ZONE, &wrap,"collections", &json);
TRI_voc_tick_t tick = TRI_CurrentTickServer();
char* tickString = TRI_StringUInt64(tick);
char const* stateStatic = "unused";
@ -3049,6 +3072,214 @@ void RestReplicationHandler::handleCommandRestoreDataCoordinator () {
TRI_DestroyJson(TRI_CORE_MEM_ZONE, &result);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief produce list of keys for a specific collection
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandCreateKeys () {
char const* collection = _request->value("collection");
if (collection == nullptr) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection parameter");
return;
}
TRI_voc_tick_t tickEnd = UINT64_MAX;
// determine end tick for keys
bool found;
char const* value = _request->value("to", found);
if (found) {
tickEnd = static_cast<TRI_voc_tick_t>(StringUtils::uint64(value));
}
TRI_vocbase_col_t* c = TRI_LookupCollectionByNameVocBase(_vocbase, collection);
if (c == nullptr) {
generateError(HttpResponse::NOT_FOUND, TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND);
return;
}
int res = TRI_ERROR_NO_ERROR;
try {
triagens::arango::CollectionGuard guard(_vocbase, c->_cid, false);
TRI_vocbase_col_t* col = guard.collection();
TRI_ASSERT(col != nullptr);
// turn off the compaction for the collection
TRI_voc_tick_t id;
res = TRI_InsertBlockerCompactorVocBase(_vocbase, 24.0 * 60.0 * 60.0, &id);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
// initialize a container with the keys
std::unique_ptr<CollectionKeys> keys(new CollectionKeys(_vocbase, col->_name, id, 300.0));
std::string const idString(std::to_string(keys->id()));
keys->create(tickEnd);
size_t const count = keys->count();
auto keysRepository = static_cast<triagens::arango::CollectionKeysRepository*>(_vocbase->_collectionKeys);
try {
keysRepository->store(keys.get());
keys.release();
}
catch (...) {
throw;
}
triagens::basics::Json json(triagens::basics::Json::Object);
json.set("id", triagens::basics::Json(idString));
json.set("count", triagens::basics::Json(static_cast<double>(count)));
generateResult(HttpResponse::OK, json.json());
}
catch (triagens::basics::Exception const& ex) {
res = ex.code();
}
catch (...) {
res = TRI_ERROR_INTERNAL;
}
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns a key range
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandGetKeys () {
std::vector<std::string> const& suffix = _request->suffix();
if (suffix.size() != 2) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expecting GET /_api/replication/keys/<keys-id>");
return;
}
std::string const& id = suffix[1];
TRI_voc_tick_t tickStart = 0;
TRI_voc_tick_t tickEnd = 0;
// determine start and end tick for keys
bool found;
char const* value = _request->value("from", found);
if (found) {
tickStart = static_cast<TRI_voc_tick_t>(StringUtils::uint64(value));
}
value = _request->value("to", found);
if (found) {
tickEnd = static_cast<TRI_voc_tick_t>(StringUtils::uint64(value));
}
if (tickStart > tickEnd || tickEnd == 0) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid from/to values");
return;
}
int res = TRI_ERROR_NO_ERROR;
try {
auto keysRepository = static_cast<triagens::arango::CollectionKeysRepository*>(_vocbase->_collectionKeys);
TRI_ASSERT(keysRepository != nullptr);
auto collectionKeysId = static_cast<triagens::arango::CollectionKeysId>(triagens::basics::StringUtils::uint64(id));
bool busy;
auto collectionKeys = keysRepository->find(collectionKeysId, busy);
if (collectionKeys == nullptr) {
if (busy) {
generateError(HttpResponse::responseCode(TRI_ERROR_CURSOR_BUSY), TRI_ERROR_CURSOR_BUSY); // TODO: Fix error code
}
else {
generateError(HttpResponse::responseCode(TRI_ERROR_CURSOR_NOT_FOUND), TRI_ERROR_CURSOR_NOT_FOUND); // TODO: fix error code
}
return;
}
try {
auto result = collectionKeys->hashChunk(tickStart, tickEnd);
collectionKeys->release();
triagens::basics::Json json(triagens::basics::Json::Object, 3);
json.set("low", triagens::basics::Json(std::get<0>(result)));
json.set("high", triagens::basics::Json(std::get<1>(result)));
json.set("hash", triagens::basics::Json(std::to_string(std::get<2>(result))));
generateResult(HttpResponse::OK, json.json());
}
catch (...) {
collectionKeys->release();
throw;
}
}
catch (triagens::basics::Exception const& ex) {
res = ex.code();
}
catch (...) {
res = TRI_ERROR_INTERNAL;
}
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
}
void RestReplicationHandler::handleCommandRemoveKeys () {
std::vector<std::string> const& suffix = _request->suffix();
if (suffix.size() != 2) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"expecting DELETE /_api/replication/keys/<keys-id>");
return;
}
std::string const& id = suffix[0];
auto keys = static_cast<triagens::arango::CollectionKeysRepository*>(_vocbase->_collectionKeys);
TRI_ASSERT(keys != nullptr);
auto collectionKeysId = static_cast<triagens::arango::CollectionKeysId>(triagens::basics::StringUtils::uint64(id));
bool found = keys->remove(collectionKeysId);
if (! found) {
generateError(HttpResponse::NOT_FOUND, TRI_ERROR_CURSOR_NOT_FOUND); // TODO: fix error code
return;
}
_response = createResponse(HttpResponse::ACCEPTED);
_response->setContentType("application/json; charset=utf-8");
triagens::basics::Json json(triagens::basics::Json::Object);
json.set("id", triagens::basics::Json(id)); // id as a string!
json.set("error", triagens::basics::Json(false));
json.set("code", triagens::basics::Json(static_cast<double>(_response->responseCode())));
json.dump(_response->body());
}
////////////////////////////////////////////////////////////////////////////////
/// @startDocuBlock JSF_get_api_replication_dump
/// @brief returns the whole content of one collection

View File

@ -298,6 +298,24 @@ namespace triagens {
void handleCommandRestoreDataCoordinator ();
////////////////////////////////////////////////////////////////////////////////
/// @brief produce list of keys for a specific collection
////////////////////////////////////////////////////////////////////////////////
void handleCommandCreateKeys ();
////////////////////////////////////////////////////////////////////////////////
/// @brief returns a key range
////////////////////////////////////////////////////////////////////////////////
void handleCommandGetKeys ();
////////////////////////////////////////////////////////////////////////////////
/// @brief remove a list of keys for a specific collection
////////////////////////////////////////////////////////////////////////////////
void handleCommandRemoveKeys ();
////////////////////////////////////////////////////////////////////////////////
/// @brief handle a dump command for a specific collection
////////////////////////////////////////////////////////////////////////////////

View File

@ -0,0 +1,205 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief collection keys container
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "Utils/CollectionKeys.h"
#include "Basics/hashes.h"
#include "Basics/JsonHelper.h"
#include "Utils/CollectionGuard.h"
#include "Utils/CollectionReadLocker.h"
#include "Utils/transactions.h"
#include "VocBase/compactor.h"
#include "VocBase/Ditch.h"
#include "VocBase/vocbase.h"
using namespace triagens::arango;
// -----------------------------------------------------------------------------
// --SECTION-- class CollectionKeys
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
// --SECTION-- constructors / destructors
// -----------------------------------------------------------------------------
CollectionKeys::CollectionKeys (TRI_vocbase_t* vocbase,
std::string const& name,
TRI_voc_tick_t blockerId,
double ttl)
: _vocbase(vocbase),
_guard(nullptr),
_document(nullptr),
_ditch(nullptr),
_name(name),
_resolver(vocbase),
_blockerId(blockerId),
_markers(nullptr),
_id(0),
_ttl(ttl),
_expires(0.0),
_isDeleted(false),
_isUsed(false) {
_id = TRI_NewTickServer();
_expires = TRI_microtime() + _ttl;
TRI_ASSERT(_blockerId > 0);
// prevent the collection from being unloaded while the export is ongoing
// this may throw
_guard = new triagens::arango::CollectionGuard(vocbase, _name.c_str(), false);
_document = _guard->collection()->_collection;
TRI_ASSERT(_document != nullptr);
}
CollectionKeys::~CollectionKeys () {
// remove compaction blocker
TRI_RemoveBlockerCompactorVocBase(_vocbase, _blockerId);
delete _markers;
if (_ditch != nullptr) {
_ditch->ditches()->freeDocumentDitch(_ditch, false);
}
delete _guard;
}
// -----------------------------------------------------------------------------
// --SECTION-- public functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief initially creates the list of keys
////////////////////////////////////////////////////////////////////////////////
void CollectionKeys::create (TRI_voc_tick_t maxTick) {
// try to acquire the exclusive lock on the compaction
while (! TRI_CheckAndLockCompactorVocBase(_document->_vocbase)) {
// didn't get it. try again...
usleep(5000);
}
// create a ditch under the compaction lock
_ditch = _document->ditches()->createDocumentDitch(false, __FILE__, __LINE__);
// release the lock
TRI_UnlockCompactorVocBase(_document->_vocbase);
// now we either have a ditch or not
if (_ditch == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
TRI_ASSERT(_markers == nullptr);
_markers = new std::vector<TRI_df_marker_t const*>();
// copy all datafile markers into the result under the read-lock
{
SingleCollectionReadOnlyTransaction trx(new StandaloneTransactionContext(), _document->_vocbase, _name);
int res = trx.begin();
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
auto idx = _document->primaryIndex();
_markers->reserve(idx->size());
triagens::basics::BucketPosition position;
uint64_t total = 0;
while (true) {
auto ptr = idx->lookupSequential(position, total);
if (ptr == nullptr) {
// done
break;
}
void const* marker = ptr->getDataPtr();
if (TRI_IsWalDataMarkerDatafile(marker)) {
continue;
}
auto df = static_cast<TRI_df_marker_t const*>(marker);
if (df->_tick >= maxTick) {
continue;
}
_markers->emplace_back(df);
}
trx.finish(res);
}
// now sort all markers without the read-lock
std::sort(_markers->begin(), _markers->end(), [] (TRI_df_marker_t const* lhs, TRI_df_marker_t const* rhs) -> bool {
int res = strcmp(TRI_EXTRACT_MARKER_KEY(lhs), TRI_EXTRACT_MARKER_KEY(rhs));
return res < 0;
});
}
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a chunk of keys
////////////////////////////////////////////////////////////////////////////////
std::tuple<std::string, std::string, uint64_t> CollectionKeys::hashChunk (size_t from, size_t to) const {
if (from >= _markers->size() || to > _markers->size() || from >= to || to == 0) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_BAD_PARAMETER);
}
std::string const first = TRI_EXTRACT_MARKER_KEY(_markers->at(from));
std::string const last = TRI_EXTRACT_MARKER_KEY(_markers->at(to - 1));
uint64_t hash = 0x012345678;
for (size_t i = from; i < to; ++i) {
auto marker = _markers->at(i);
char const* key = TRI_EXTRACT_MARKER_KEY(marker);
hash ^= TRI_FnvHashString(key);
hash ^= TRI_EXTRACT_MARKER_RID(marker);
}
return std::make_tuple(first, last, hash);
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:

View File

@ -0,0 +1,160 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief collection keys
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_ARANGO_COLLECTION_KEYS_H
#define ARANGODB_ARANGO_COLLECTION_KEYS_H 1
#include "Basics/Common.h"
#include "Utils/CollectionNameResolver.h"
#include "VocBase/voc-types.h"
struct TRI_document_collection_t;
struct TRI_vocbase_t;
namespace triagens {
namespace arango {
class CollectionGuard;
class DocumentDitch;
typedef TRI_voc_tick_t CollectionKeysId;
// -----------------------------------------------------------------------------
// --SECTION-- class CollectionKeys
// -----------------------------------------------------------------------------
class CollectionKeys {
public:
CollectionKeys (CollectionKeys const&) = delete;
CollectionKeys& operator= (CollectionKeys const&) = delete;
CollectionKeys (TRI_vocbase_t*,
std::string const&,
TRI_voc_tick_t,
double ttl);
~CollectionKeys ();
// -----------------------------------------------------------------------------
// --SECTION-- public functions
// -----------------------------------------------------------------------------
public:
CollectionKeysId id () const {
return _id;
}
double ttl () const {
return _ttl;
}
double expires () const {
return _expires;
}
bool isUsed () const {
return _isUsed;
}
bool isDeleted () const {
return _isDeleted;
}
void deleted () {
_isDeleted = true;
}
void use () {
TRI_ASSERT(! _isDeleted);
TRI_ASSERT(! _isUsed);
_isUsed = true;
_expires = TRI_microtime() + _ttl;
}
void release () {
TRI_ASSERT(_isUsed);
_isUsed = false;
}
size_t count () const {
TRI_ASSERT(_markers != nullptr);
return _markers->size();
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initially creates the list of keys
////////////////////////////////////////////////////////////////////////////////
void create (TRI_voc_tick_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a chunk of keys
////////////////////////////////////////////////////////////////////////////////
std::tuple<std::string, std::string, uint64_t> hashChunk (size_t, size_t) const;
// -----------------------------------------------------------------------------
// --SECTION-- private variables
// -----------------------------------------------------------------------------
private:
struct TRI_vocbase_t* _vocbase;
triagens::arango::CollectionGuard* _guard;
struct TRI_document_collection_t* _document;
triagens::arango::DocumentDitch* _ditch;
std::string const _name;
triagens::arango::CollectionNameResolver _resolver;
TRI_voc_tick_t _blockerId;
std::vector<TRI_df_marker_t const*>* _markers;
CollectionKeysId _id;
double _ttl;
double _expires;
bool _isDeleted;
bool _isUsed;
};
}
}
#endif
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:

View File

@ -0,0 +1,295 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief list of collection keys present in database
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "Utils/CollectionKeysRepository.h"
#include "Basics/json.h"
#include "Basics/logging.h"
#include "Basics/MutexLocker.h"
#include "VocBase/server.h"
#include "VocBase/vocbase.h"
using namespace triagens::arango;
// -----------------------------------------------------------------------------
// --SECTION-- CollectionKeysRepository
// -----------------------------------------------------------------------------
size_t const CollectionKeysRepository::MaxCollectCount = 32;
// -----------------------------------------------------------------------------
// --SECTION-- constructors / destructors
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief create a collection keys repository
////////////////////////////////////////////////////////////////////////////////
CollectionKeysRepository::CollectionKeysRepository (TRI_vocbase_t* vocbase)
: _vocbase(vocbase),
_lock(),
_keys() {
_keys.reserve(64);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a collection keys repository
////////////////////////////////////////////////////////////////////////////////
CollectionKeysRepository::~CollectionKeysRepository () {
try {
garbageCollect(true);
}
catch (...) {
}
// wait until all used entries have vanished
int tries = 0;
while (true) {
if (! containsUsed()) {
break;
}
if (tries == 0) {
LOG_INFO("waiting for used keys to become unused");
}
else if (tries == 120) {
LOG_WARNING("giving up waiting for unused keys");
}
usleep(500000);
++tries;
}
{
MUTEX_LOCKER(_lock);
for (auto it : _keys) {
delete it.second;
}
_keys.clear();
}
}
// -----------------------------------------------------------------------------
// --SECTION-- public methods
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief stores collection keys in the repository
////////////////////////////////////////////////////////////////////////////////
void CollectionKeysRepository::store (triagens::arango::CollectionKeys* keys) {
MUTEX_LOCKER(_lock);
_keys.emplace(keys->id(), keys);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief remove collection keys by id
////////////////////////////////////////////////////////////////////////////////
bool CollectionKeysRepository::remove (CollectionKeysId id) {
triagens::arango::CollectionKeys* collectionKeys = nullptr;
{
MUTEX_LOCKER(_lock);
auto it = _keys.find(id);
if (it == _keys.end()) {
// not found
return false;
}
collectionKeys = (*it).second;
if (collectionKeys->isDeleted()) {
// already deleted
return false;
}
if (collectionKeys->isUsed()) {
// keys are in use by someone else. now mark as deleted
collectionKeys->deleted();
return true;
}
// keys are not in use by someone else
_keys.erase(it);
}
TRI_ASSERT(collectionKeys != nullptr);
delete collectionKeys;
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief find an existing keys entry by id
/// if found, the keys will be returned with the usage flag set to true.
/// they must be returned later using release()
////////////////////////////////////////////////////////////////////////////////
CollectionKeys* CollectionKeysRepository::find (CollectionKeysId id,
bool& busy) {
triagens::arango::CollectionKeys* collectionKeys = nullptr;
busy = false;
{
MUTEX_LOCKER(_lock);
auto it = _keys.find(id);
if (it == _keys.end()) {
// not found
return nullptr;
}
collectionKeys = (*it).second;
if (collectionKeys->isDeleted()) {
// already deleted
return nullptr;
}
if (collectionKeys->isUsed()) {
busy = true;
return nullptr;
}
collectionKeys->use();
}
return collectionKeys;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief return collection keys
////////////////////////////////////////////////////////////////////////////////
void CollectionKeysRepository::release (CollectionKeys* collectionKeys) {
{
MUTEX_LOCKER(_lock);
TRI_ASSERT(collectionKeys->isUsed());
collectionKeys->release();
if (! collectionKeys->isDeleted()) {
return;
}
// remove from the list
_keys.erase(collectionKeys->id());
}
// and free the keys
delete collectionKeys;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not the repository contains a used keys entry
////////////////////////////////////////////////////////////////////////////////
bool CollectionKeysRepository::containsUsed () {
MUTEX_LOCKER(_lock);
for (auto it : _keys) {
if (it.second->isUsed()) {
return true;
}
}
return false;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief run a garbage collection on the data
////////////////////////////////////////////////////////////////////////////////
bool CollectionKeysRepository::garbageCollect (bool force) {
std::vector<triagens::arango::CollectionKeys*> found;
found.reserve(MaxCollectCount);
auto const now = TRI_microtime();
{
MUTEX_LOCKER(_lock);
for (auto it = _keys.begin(); it != _keys.end(); /* no hoisting */) {
auto collectionKeys = (*it).second;
if (collectionKeys->isUsed()) {
// must not destroy anything currently in use
++it;
continue;
}
if (force || collectionKeys->expires() < now) {
collectionKeys->deleted();
}
if (collectionKeys->isDeleted()) {
try {
found.emplace_back(collectionKeys);
it = _keys.erase(it);
}
catch (...) {
// stop iteration
break;
}
if (! force &&
found.size() >= MaxCollectCount) {
break;
}
}
else {
++it;
}
}
}
// remove keys outside the lock
for (auto it : found) {
delete it;
}
return (! found.empty());
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:

View File

@ -0,0 +1,158 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief list of collection keys dumps present in database
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2012-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGODB_ARANGO_COLLECTION_KEYS_REPOSITORY_H
#define ARANGODB_ARANGO_COLLECTION_KEYS_REPOSITORY_H 1
#include "Basics/Common.h"
#include "Basics/Mutex.h"
#include "Utils/CollectionKeys.h"
#include "VocBase/voc-types.h"
struct TRI_json_t;
struct TRI_vocbase_t;
namespace triagens {
namespace arango {
class CollectionKeys;
// -----------------------------------------------------------------------------
// --SECTION-- class CollectionKeysRepository
// -----------------------------------------------------------------------------
class CollectionKeysRepository {
// -----------------------------------------------------------------------------
// --SECTION-- constructors / destructors
// -----------------------------------------------------------------------------
public:
////////////////////////////////////////////////////////////////////////////////
/// @brief create a collection keys repository
////////////////////////////////////////////////////////////////////////////////
explicit CollectionKeysRepository (TRI_vocbase_t*);
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a collection keys repository
////////////////////////////////////////////////////////////////////////////////
~CollectionKeysRepository ();
// -----------------------------------------------------------------------------
// --SECTION-- public methods
// -----------------------------------------------------------------------------
public:
////////////////////////////////////////////////////////////////////////////////
/// @brief stores collection keys in the repository
////////////////////////////////////////////////////////////////////////////////
void store (CollectionKeys*);
////////////////////////////////////////////////////////////////////////////////
/// @brief remove a keys by id
////////////////////////////////////////////////////////////////////////////////
bool remove (CollectionKeysId);
////////////////////////////////////////////////////////////////////////////////
/// @brief find an existing collection keys list by id
/// if found, the keys will be returned with the usage flag set to true.
/// it must be returned later using release()
////////////////////////////////////////////////////////////////////////////////
CollectionKeys* find (CollectionKeysId,
bool&);
////////////////////////////////////////////////////////////////////////////////
/// @brief return a cursor
////////////////////////////////////////////////////////////////////////////////
void release (CollectionKeys*);
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not the repository contains used keys
////////////////////////////////////////////////////////////////////////////////
bool containsUsed ();
////////////////////////////////////////////////////////////////////////////////
/// @brief run a garbage collection on the data
////////////////////////////////////////////////////////////////////////////////
bool garbageCollect (bool);
// -----------------------------------------------------------------------------
// --SECTION-- private variables
// -----------------------------------------------------------------------------
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief vocbase
////////////////////////////////////////////////////////////////////////////////
TRI_vocbase_t* _vocbase;
////////////////////////////////////////////////////////////////////////////////
/// @brief mutex for the repository
////////////////////////////////////////////////////////////////////////////////
triagens::basics::Mutex _lock;
////////////////////////////////////////////////////////////////////////////////
/// @brief list of current keys
////////////////////////////////////////////////////////////////////////////////
std::unordered_map<CollectionKeysId, CollectionKeys*> _keys;
////////////////////////////////////////////////////////////////////////////////
/// @brief maximum number of keys to garbage-collect in one go
////////////////////////////////////////////////////////////////////////////////
static size_t const MaxCollectCount;
};
}
}
#endif
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------
// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:

View File

@ -47,6 +47,7 @@
#include "Basics/tri-strings.h"
#include "Basics/threads.h"
#include "Basics/Exceptions.h"
#include "Utils/CollectionKeysRepository.h"
#include "Utils/CursorRepository.h"
#include "Utils/transactions.h"
#include "VocBase/auth.h"
@ -1508,6 +1509,11 @@ void TRI_DestroyVocBase (TRI_vocbase_t* vocbase) {
if (vocbase->_cursorRepository != nullptr) {
vocbase->_cursorRepository->garbageCollect(true);
}
// mark all collection keys as deleted so underlying collections can be freed soon
if (vocbase->_collectionKeys != nullptr) {
vocbase->_collectionKeys->garbageCollect(true);
}
std::vector<TRI_vocbase_col_t*> collections;
@ -2381,14 +2387,16 @@ TRI_vocbase_t::TRI_vocbase_t (TRI_server_t* server,
_userStructures(nullptr),
_queries(nullptr),
_cursorRepository(nullptr),
_collectionKeys(nullptr),
_authInfoLoaded(false),
_hasCompactor(false),
_isOwnAppsDirectory(true),
_oldTransactions(nullptr),
_replicationApplier(nullptr) {
_queries = new triagens::aql::QueryList(this);
_queries = new triagens::aql::QueryList(this);
_cursorRepository = new triagens::arango::CursorRepository(this);
_collectionKeys = new triagens::arango::CollectionKeysRepository(this);
_path = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE, path);
_name = TRI_DuplicateStringZ(TRI_CORE_MEM_ZONE, name);
@ -2447,6 +2455,7 @@ TRI_vocbase_t::~TRI_vocbase_t () {
TRI_DestroyAssociativePointer(&_collectionsById);
delete _cursorRepository;
delete _collectionKeys;
delete _queries;
// free name and path

View File

@ -56,6 +56,7 @@ namespace triagens {
class QueryList;
}
namespace arango {
class CollectionKeysRepository;
class CursorRepository;
}
}
@ -286,6 +287,7 @@ struct TRI_vocbase_t {
void* _userStructures;
triagens::aql::QueryList* _queries;
triagens::arango::CursorRepository* _cursorRepository;
triagens::arango::CollectionKeysRepository* _collectionKeys;
TRI_associative_pointer_t _authInfo;
TRI_associative_pointer_t _authCache;