Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/arangodb/arangodb into devel

This commit is contained in:
jsteemann 2017-01-09 10:59:15 +01:00
parent 503326852c 6830fd57f8
commit f69985014b
4 changed files with 97 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
arangod/Agency/Supervision.cpp
View File

@ -935,4 +935,26 @@ void Supervision::beginShutdown() {
guard.broadcast();
}
Store const& Supervision::store() const { return _agent->readDB(); }
void Supervision::missingPrototype() {
auto const& plannedDBs = _snapshot(planColPrefix).children();
auto available = Job::availableServers(_snapshot);
// key: prototype, value: clone
std::multimap<std::string, std::string> likeness;
for (const auto& db_ : plannedDBs) { // Planned databases
auto const& db = *(db_.second);
for (const auto& col_ : db.children()) { // Planned collections
auto const& col = *(col_.second);
auto prototype = col("distributeShardsLike").slice().copyString();
if (prototype.empty()) {
continue;
}
}
}
}

6
arangod/Agency/Supervision.h
View File

@ -116,6 +116,9 @@ class Supervision : public arangodb::Thread {
private:
/// @brief Check for inconsistencies in distributeShardsLike
void missingPrototype();
/// @brief Check for inconsistencies in replication factor vs dbs entries
void enforceReplication();
@ -141,9 +144,6 @@ class Supervision : public arangodb::Thread {
/// @brief Get unique ids from agency
void getUniqueIds();
/// @brief Read db
Store const& store() const;
/// @brief Perform sanity checking
bool doChecks();

141
js/server/modules/@arangodb/cluster.js
View File

@ -38,6 +38,14 @@ var wait = require('internal').wait;
var isEnterprise = require('internal').isEnterprise();
var _ = require('lodash');
const curDatabases = '/arango/Current/Databases/';
const curCollections = '/arango/Current/Collections/';
const curVersion = '/arango/Current/Version';
const agencyOperations = {
'delete' : {'op' : 'delete'},
'increment' : {'op' : 'increment'}
};
var endpointToURL = function (endpoint) {
if (endpoint.substr(0, 6) === 'ssl://') {
return 'https://' + endpoint.substr(6);
@ -301,11 +309,13 @@ function getLocalCollections () {
// / @brief create databases if they exist in the plan but not locally
// //////////////////////////////////////////////////////////////////////////////
function createLocalDatabases (plannedDatabases, currentDatabases, writeLocked) {
function createLocalDatabases (plannedDatabases, currentDatabases) {
var ourselves = global.ArangoServerState.id();
var createDatabaseAgency = function (payload) {
global.ArangoAgency.set('Current/Databases/' + payload.name + '/' + ourselves,
payload);
var envelope = {};
envelope[curDatabases + payload.name + '/' + ourselves] = payload;
envelope[curVersion] = {"op":"increment"};
global.ArangoAgency.write([[envelope]]);
};
var db = require('internal').db;
@ -339,15 +349,11 @@ function createLocalDatabases (plannedDatabases, currentDatabases, writeLocked)
payload.errorNum = err.errorNum;
payload.errorMessage = err.errorMessage;
}
writeLocked({ part: 'Current' },
createDatabaseAgency,
[ payload ]);
createDatabaseAgency(payload);
} else if (typeof currentDatabases[name] !== 'object' || !currentDatabases[name].hasOwnProperty(ourselves)) {
// mop: ok during cluster startup we have this buggy situation where a dbserver
// has a database but has not yet announced it to the agency :S
writeLocked({ part: 'Current' },
createDatabaseAgency,
[ payload ]);
createDatabaseAgency(payload);
}
}
}
@ -357,13 +363,16 @@ function createLocalDatabases (plannedDatabases, currentDatabases, writeLocked)
// / @brief drop databases if they do exist locally but not in the plan
// //////////////////////////////////////////////////////////////////////////////
function dropLocalDatabases (plannedDatabases, writeLocked) {
function dropLocalDatabases (plannedDatabases) {
var ourselves = global.ArangoServerState.id();
var dropDatabaseAgency = function (payload) {
try {
global.ArangoAgency.remove('Current/Databases/' + payload.name + '/' + ourselves);
} catch (err) {
var envelope = {};
envelope[curDatabases + payload.name + '/' + ourselves] = {'op':'delete'};
envelope[curVersion] = {'op':'increment'};
global.ArangoAgency.write([[envelope]]);
} catch (err) {s
// ignore errors
}
};
@ -399,9 +408,7 @@ function dropLocalDatabases (plannedDatabases, writeLocked) {
}
db._dropDatabase(name);
writeLocked({ part: 'Current' },
dropDatabaseAgency,
[ { name: name } ]);
dropDatabaseAgency({name: name});
}
}
}
@ -411,12 +418,15 @@ function dropLocalDatabases (plannedDatabases, writeLocked) {
// / @brief clean up what's in Current/Databases for ourselves
// //////////////////////////////////////////////////////////////////////////////
function cleanupCurrentDatabases (currentDatabases, writeLocked) {
function cleanupCurrentDatabases (currentDatabases) {
var ourselves = global.ArangoServerState.id();
var dropDatabaseAgency = function (payload) {
try {
global.ArangoAgency.remove('Current/Databases/' + payload.name + '/' + ourselves);
var envelope = {};
envelope[curDatabases + payload.name + '/' + ourselves] = {'op':'delete'};
envelope[curVersion] = {'op':'increment'};
global.ArangoAgency.write([[envelope]]);
} catch (err) {
// ignore errors
}
@ -437,9 +447,7 @@ function cleanupCurrentDatabases (currentDatabases, writeLocked) {
// we are entered for a database that we don't have locally
console.debug("cleaning up entry for unknown database '%s'", name);
writeLocked({ part: 'Current' },
dropDatabaseAgency,
[ { name: name } ]);
dropDatabaseAgency({name: name});
}
}
}
@ -450,13 +458,13 @@ function cleanupCurrentDatabases (currentDatabases, writeLocked) {
// / @brief handle database changes
// //////////////////////////////////////////////////////////////////////////////
function handleDatabaseChanges (plan, current, writeLocked) {
function handleDatabaseChanges (plan, current) {
var plannedDatabases = plan.Databases;
var currentDatabases = current.Databases;
createLocalDatabases(plannedDatabases, currentDatabases, writeLocked);
dropLocalDatabases(plannedDatabases, writeLocked);
cleanupCurrentDatabases(currentDatabases, writeLocked);
createLocalDatabases(plannedDatabases, currentDatabases);
dropLocalDatabases(plannedDatabases);
cleanupCurrentDatabases(currentDatabases);
}
// //////////////////////////////////////////////////////////////////////////////
@ -464,8 +472,7 @@ function handleDatabaseChanges (plan, current, writeLocked) {
// //////////////////////////////////////////////////////////////////////////////
function createLocalCollections (plannedCollections, planVersion,
currentCollections,
takeOverResponsibility, writeLocked) {
currentCollections, takeOverResponsibility) {
var ourselves = global.ArangoServerState.id();
var createCollectionAgency = function (database, shard, collInfo, error) {
@ -479,9 +486,13 @@ function createLocalCollections (plannedCollections, planVersion,
console.debug('creating Current/Collections/' + database + '/' +
collInfo.planId + '/' + shard);
global.ArangoAgency.set('Current/Collections/' + database + '/' +
collInfo.planId + '/' + shard,
payload);
var envelope = {};
//envelope[curCollections + database + '/' + collInfo.planId + '/' + shard] = payload;
global.ArangoAgency.set('Current/Collections/' + database + '/' + collInfo.planId + '/' + shard, payload);
envelope[curVersion] = {'op':'increment'};
global.ArangoAgency.write([[envelope]]);
console.debug('creating Current/Collections/' + database + '/' +
collInfo.planId + '/' + shard + ' done.');
};
@ -562,9 +573,7 @@ function createLocalCollections (plannedCollections, planVersion,
}
if (isLeader) {
writeLocked({ part: 'Current' },
createCollectionAgency,
[ database, shard, collInfo, error ]);
createCollectionAgency(database, shard, collInfo, error);
didWrite = true;
}
} else {
@ -589,9 +598,7 @@ function createLocalCollections (plannedCollections, planVersion,
db._collection(shard).load();
}
if (isLeader) {
writeLocked({ part: 'Current' },
createCollectionAgency,
[ database, shard, collInfo, error ]);
createCollectionAgency(database, shard, collInfo, error);
didWrite = true;
}
}
@ -620,9 +627,7 @@ function createLocalCollections (plannedCollections, planVersion,
errorMessage: err3.errorMessage };
}
if (isLeader) {
writeLocked({ part: 'Current' },
createCollectionAgency,
[ database, shard, collInfo, error ]);
createCollectionAgency(database, shard, collInfo, error);
didWrite = true;
}
}
@ -631,9 +636,7 @@ function createLocalCollections (plannedCollections, planVersion,
if (error.error) {
if (takeOverResponsibility && !didWrite) {
if (isLeader) {
writeLocked({ part: 'Current' },
takeOver,
[ database, shard, collInfo, error ]);
takeOver(database, shard, collInfo, error);
}
}
continue; // No point to look for properties and
@ -671,9 +674,7 @@ function createLocalCollections (plannedCollections, planVersion,
changed = true;
}
if (changed && isLeader) {
writeLocked({ part: 'Current' },
createCollectionAgency,
[ database, shard, collInfo, error ]);
createCollectionAgency(database, shard, collInfo, error);
didWrite = true;
}
}
@ -711,18 +712,14 @@ function createLocalCollections (plannedCollections, planVersion,
}
}
if (changed2 && isLeader) {
writeLocked({ part: 'Current' },
createCollectionAgency,
[ database, shard, collInfo, error ]);
createCollectionAgency(database, shard, collInfo, error);
didWrite = true;
}
}
if ((takeOverResponsibility && !didWrite && isLeader) ||
(!didWrite && isLeader && !wasLeader)) {
writeLocked({ part: 'Current' },
takeOver,
[ database, shard, collInfo, error ]);
takeOver(database, shard, collInfo, error);
}
}
}
@ -754,7 +751,7 @@ function createLocalCollections (plannedCollections, planVersion,
};
writeLocked({ part: 'Current' }, migrate, [fakeLock]);
} else {
migrate(writeLocked);
migrate();
}
}
@ -785,15 +782,18 @@ function leaderResign (database, collId, shardName, ourselves) {
// / @brief drop collections if they exist locally but not in the plan
// //////////////////////////////////////////////////////////////////////////////
function dropLocalCollections (plannedCollections, currentCollections,
writeLocked) {
function dropLocalCollections (plannedCollections, currentCollections) {
var ourselves = global.ArangoServerState.id();
var dropCollectionAgency = function (database, shardID, id) {
try {
console.debug('dropping Current/Collections/' + database + '/' +
id + '/' + shardID);
global.ArangoAgency.remove('Current/Collections/' + database + '/' + id + '/' + shardID);
var envelope = {};
envelope[curCollections + database + '/' + id + '/' + shardID] =
{'op':'delete'};
envelope[curVersion] = {'op':'increment'};
global.ArangoAgency.write([[envelope]]);
console.debug('dropping Current/Collections/' + database + '/' +
id + '/' + shardID + ' done.');
} catch (err) {
@ -876,9 +876,7 @@ function dropLocalCollections (plannedCollections, currentCollections,
console.debug('cleaning out Current entry for shard %s in',
'agency for %s/%s', collection, database,
collections[collection].name);
writeLocked({ part: 'Current' },
dropCollectionAgency,
[ database, collection, collections[collection].planId ]);
dropCollectionAgency(database, collection, collections[collection].planId);
}
}
}
@ -897,13 +895,16 @@ function dropLocalCollections (plannedCollections, currentCollections,
// / @brief clean up what's in Current/Collections for ourselves
// //////////////////////////////////////////////////////////////////////////////
function cleanupCurrentCollections (plannedCollections, currentCollections,
writeLocked) {
function cleanupCurrentCollections (plannedCollections, currentCollections) {
var dropCollectionAgency = function (database, collection, shardID) {
try {
console.debug('cleaning Current/Collections/' + database + '/' +
collection + '/' + shardID);
global.ArangoAgency.remove('Current/Collections/' + database + '/' + collection + '/' + shardID);
var envelope = {};
envelope[curCollections + database + '/' + collection + '/' + shardID] =
{'op':'delete'};
envelope[curVersion] = {'op':'increment'};
global.ArangoAgency.write([[envelope]]);
console.debug('cleaning Current/Collections/' + database + '/' +
collection + '/' + shardID + ' done.');
} catch (err) {
@ -937,9 +938,7 @@ function cleanupCurrentCollections (plannedCollections, currentCollections,
database,
collection);
writeLocked({ part: 'Current' },
dropCollectionAgency,
[ database, collection, shard ]);
dropCollectionAgency(database, collection, shard);
}
}
}
@ -1294,8 +1293,7 @@ function synchronizeLocalFollowerCollections (plannedCollections,
// / @brief handle collection changes
// //////////////////////////////////////////////////////////////////////////////
function handleCollectionChanges (plan, current, takeOverResponsibility,
writeLocked) {
function handleCollectionChanges (plan, current, takeOverResponsibility) {
var plannedCollections = plan.Collections;
var currentCollections = current.Collections;
@ -1303,10 +1301,9 @@ function handleCollectionChanges (plan, current, takeOverResponsibility,
try {
createLocalCollections(plannedCollections, plan.Version, currentCollections,
takeOverResponsibility, writeLocked);
dropLocalCollections(plannedCollections, currentCollections, writeLocked);
cleanupCurrentCollections(plannedCollections, currentCollections,
writeLocked);
takeOverResponsibility);
dropLocalCollections(plannedCollections, currentCollections);
cleanupCurrentCollections(plannedCollections, currentCollections);
if (!synchronizeLocalFollowerCollections(plannedCollections,
currentCollections)) {
// If not all needed jobs have been scheduled, then work is still
@ -1407,7 +1404,7 @@ function primaryToSecondary () {
// / @brief change handling trampoline function
// //////////////////////////////////////////////////////////////////////////////
function handleChanges (plan, current, writeLocked) {
function handleChanges (plan, current) {
var changed = false;
var role = ArangoServerState.role();
if (role === 'PRIMARY' || role === 'SECONDARY') {
@ -1452,12 +1449,12 @@ function handleChanges (plan, current, writeLocked) {
}
}
handleDatabaseChanges(plan, current, writeLocked);
handleDatabaseChanges(plan, current);
var success;
if (role === 'PRIMARY' || role === 'COORDINATOR') {
// Note: This is only ever called for DBservers (primary and secondary),
// we keep the coordinator case here just in case...
success = handleCollectionChanges(plan, current, changed, writeLocked);
success = handleCollectionChanges(plan, current, changed);
} else {
success = setupReplication();
}

6
js/server/tests/shell/shell-database-noncluster.js
View File

@ -203,10 +203,8 @@ function DatabaseSuite () {
if (tries > 15) {
require("internal").printf("[WARNING] waited " + tries * 2 +" seconds for " + path + " do disappear");
}
// if (require("internal").platform.substr(0,3) !== 'win') {
// yes, we know this test fails in windows now and then.
assertFalse(fs.exists(path));
//}
// yes, we know this test fails in windows now and then.
assertFalse(fs.exists(path));
}
};