Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

updated arangorestore

This commit is contained in:
Jan Steemann 2013-10-10 16:58:21 +02:00
parent 6a353a4c38
commit b8318609e2
10 changed files with 535 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
arangod/Replication/InitialSyncer.cpp
View File

@ -593,10 +593,10 @@ int InitialSyncer::handleCollectionDump (TRI_transaction_collection_t* trxCollec
/// @brief handle the information about a collection
////////////////////////////////////////////////////////////////////////////////
int InitialSyncer::handleCollectionInitial (TRI_json_t const* parameters,
TRI_json_t const* indexes,
string& errorMsg,
sync_phase_e phase) {
int InitialSyncer::handleCollection (TRI_json_t const* parameters,
TRI_json_t const* indexes,
string& errorMsg,
sync_phase_e phase) {
sendExtendBatch();
@ -750,6 +750,13 @@ int InitialSyncer::handleCollectionInitial (TRI_json_t const* parameters,
res = handleCollectionDump(trxCollection, masterName, _masterInfo._state._lastLogTick, errorMsg);
}
if (res == TRI_ERROR_NO_ERROR) {
TRI_CommitTransaction(trx, TRI_TRANSACTION_TOP_LEVEL);
}
TRI_FreeTransaction(trx);
if (res == TRI_ERROR_NO_ERROR) {
// now create indexes
const size_t n = indexes->_value._objects._length;
@ -758,39 +765,51 @@ int InitialSyncer::handleCollectionInitial (TRI_json_t const* parameters,
const string progress = "creating indexes for " + collectionMsg;
setProgress(progress.c_str());
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* idxDef = (TRI_json_t const*) TRI_AtVector(&indexes->_value._objects, i);
TRI_index_t* idx = 0;
// {"id":"229907440927234","type":"hash","unique":false,"fields":["x","Y"]}
res = TRI_FromJsonIndexDocumentCollection((TRI_document_collection_t*) trxCollection->_collection->_collection, idxDef, &idx);
TRI_ReadLockReadWriteLock(&_vocbase->_inventoryLock);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "could not create index: " + string(TRI_errno_string(res));
break;
}
else {
assert(idx != 0);
TRI_vocbase_col_t* col = TRI_UseCollectionByIdVocBase(_vocbase, cid);
res = TRI_SaveIndex((TRI_primary_collection_t*) trxCollection->_collection->_collection,
idx,
_masterInfo._serverId);
if (col == 0) {
res = TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
}
else {
TRI_primary_collection_t* primary = col->_collection;
assert(primary != 0);
TRI_WRITE_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* idxDef = (TRI_json_t const*) TRI_AtVector(&indexes->_value._objects, i);
TRI_index_t* idx = 0;
// {"id":"229907440927234","type":"hash","unique":false,"fields":["x","Y"]}
res = TRI_FromJsonIndexDocumentCollection((TRI_document_collection_t*) primary, idxDef, &idx);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "could not save index: " + string(TRI_errno_string(res));
errorMsg = "could not create index: " + string(TRI_errno_string(res));
break;
}
else {
assert(idx != 0);
res = TRI_SaveIndex(primary, idx, _masterInfo._serverId);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "could not save index: " + string(TRI_errno_string(res));
break;
}
}
}
TRI_WRITE_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
TRI_ReleaseCollectionVocBase(_vocbase, col);
}
TRI_ReadUnlockReadWriteLock(&_vocbase->_inventoryLock);
}
}
if (res == TRI_ERROR_NO_ERROR) {
TRI_CommitTransaction(trx, TRI_TRANSACTION_TOP_LEVEL);
}
TRI_FreeTransaction(trx);
return res;
}
@ -890,7 +909,7 @@ int InitialSyncer::iterateCollections (TRI_json_t const* collections,
return TRI_ERROR_REPLICATION_INVALID_RESPONSE;
}
int res = handleCollectionInitial(parameters, indexes, errorMsg, phase);
int res = handleCollection(parameters, indexes, errorMsg, phase);
if (res != TRI_ERROR_NO_ERROR) {
return res;

8
arangod/Replication/InitialSyncer.h
View File

@ -224,10 +224,10 @@ namespace triagens {
/// @brief handle the information about a collection
////////////////////////////////////////////////////////////////////////////////
int handleCollectionInitial (struct TRI_json_s const*,
struct TRI_json_s const*,
std::string&,
sync_phase_e);
int handleCollection (struct TRI_json_s const*,
struct TRI_json_s const*,
std::string&,
sync_phase_e);
////////////////////////////////////////////////////////////////////////////////
/// @brief handle the inventory response of the master

342
arangod/RestHandler/RestReplicationHandler.cpp
View File

@ -164,6 +164,12 @@ Handler::status_e RestReplicationHandler::execute() {
}
handleCommandRestoreCollection();
}
else if (command == "restore-indexes") {
if (type != HttpRequest::HTTP_REQUEST_PUT) {
goto BAD_CALL;
}
handleCommandRestoreIndexes();
}
else if (command == "restore-data") {
if (type != HttpRequest::HTTP_REQUEST_PUT) {
goto BAD_CALL;
@ -1423,49 +1429,6 @@ void RestReplicationHandler::handleCommandInventory () {
TRI_DestroyJson(TRI_CORE_MEM_ZONE, &json);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection TODO
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandRestoreCollection () {
TRI_json_t* json = _request->toJson(0);
if (json == 0) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid JSON");
return;
}
bool found;
char const* value;
bool overwrite = false;
value = _request->value("overwrite", found);
if (found) {
overwrite = StringUtils::boolean(value);
}
TRI_server_id_t remoteServerId = 0; // TODO
string errorMsg;
int res = processRestoreCollection(json, overwrite, remoteServerId, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
else {
TRI_json_t result;
TRI_InitArrayJson(TRI_CORE_MEM_ZONE, &result);
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, &result, "result", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, true));
generateResult(&result);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief extract the collection id from JSON TODO: move
////////////////////////////////////////////////////////////////////////////////
@ -1493,6 +1456,7 @@ TRI_voc_cid_t RestReplicationHandler::getCid (TRI_json_t const* json) const {
int RestReplicationHandler::createCollection (TRI_json_t const* json,
TRI_vocbase_col_t** dst,
bool reuseId,
TRI_server_id_t remoteServerId) {
if (dst != 0) {
*dst = 0;
@ -1508,15 +1472,23 @@ int RestReplicationHandler::createCollection (TRI_json_t const* json,
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
const TRI_voc_cid_t cid = getCid(json);
TRI_voc_cid_t cid = 0;
if (cid == 0) {
return TRI_ERROR_HTTP_BAD_PARAMETER;
if (reuseId) {
cid = getCid(json);
if (cid == 0) {
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
}
const TRI_col_type_e type = (TRI_col_type_e) JsonHelper::getNumericValue<int>(json, "type", (int) TRI_COL_TYPE_DOCUMENT);
TRI_vocbase_col_t* col = TRI_LookupCollectionByIdVocBase(_vocbase, cid);
TRI_vocbase_col_t* col = 0;
if (cid > 0) {
col = TRI_LookupCollectionByIdVocBase(_vocbase, cid);
}
if (col != 0 &&
(TRI_col_type_t) col->_type == (TRI_col_type_t) type) {
@ -1547,27 +1519,29 @@ int RestReplicationHandler::createCollection (TRI_json_t const* json,
params._waitForSync = JsonHelper::getBooleanValue(json, "waitForSync", _vocbase->_settings.defaultWaitForSync);
params._isVolatile = JsonHelper::getBooleanValue(json, "isVolatile", false);
// wait for "old" collection to be dropped
char* dirName = TRI_GetDirectoryCollection(_vocbase->_path,
name.c_str(),
type,
cid);
if (cid > 0) {
// wait for "old" collection to be dropped
char* dirName = TRI_GetDirectoryCollection(_vocbase->_path,
name.c_str(),
type,
cid);
if (dirName != 0) {
char* parameterName = TRI_Concatenate2File(dirName, TRI_VOC_PARAMETER_FILE);
if (dirName != 0) {
char* parameterName = TRI_Concatenate2File(dirName, TRI_VOC_PARAMETER_FILE);
if (parameterName != 0) {
int iterations = 0;
if (parameterName != 0) {
int iterations = 0;
// TODO: adjust sleep timer & maxiterations
while (TRI_IsDirectory(dirName) && TRI_ExistsFile(parameterName) && iterations++ < 1200) {
usleep(100 * 1000);
// TODO: adjust sleep timer & maxiterations
while (TRI_IsDirectory(dirName) && TRI_ExistsFile(parameterName) && iterations++ < 1200) {
usleep(100 * 1000);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, parameterName);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, parameterName);
TRI_FreeString(TRI_CORE_MEM_ZONE, dirName);
}
TRI_FreeString(TRI_CORE_MEM_ZONE, dirName);
}
col = TRI_CreateCollectionVocBase(_vocbase, &params, cid, remoteServerId);
@ -1584,12 +1558,95 @@ int RestReplicationHandler::createCollection (TRI_json_t const* json,
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection TODO
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandRestoreCollection () {
TRI_json_t* json = _request->toJson(0);
if (json == 0) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid JSON");
return;
}
bool found;
char const* value;
bool overwrite = false;
value = _request->value("overwrite", found);
if (found) {
overwrite = StringUtils::boolean(value);
}
bool recycleIds = false;
value = _request->value("recycleIds", found);
if (found) {
recycleIds = StringUtils::boolean(value);
}
TRI_server_id_t remoteServerId = 0; // TODO
string errorMsg;
int res = processRestoreCollection(json, overwrite, recycleIds, remoteServerId, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
else {
TRI_json_t result;
TRI_InitArrayJson(TRI_CORE_MEM_ZONE, &result);
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, &result, "result", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, true));
generateResult(&result);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the indexes of a collection TODO
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandRestoreIndexes () {
TRI_json_t* json = _request->toJson(0);
if (json == 0) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid JSON");
return;
}
TRI_server_id_t remoteServerId = 0; // TODO
string errorMsg;
int res = processRestoreIndexes(json, remoteServerId, errorMsg);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
else {
TRI_json_t result;
TRI_InitArrayJson(TRI_CORE_MEM_ZONE, &result);
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, &result, "result", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, true));
generateResult(&result);
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collection,
bool dropExisting,
bool reuseId,
TRI_server_id_t remoteServerId,
string& errorMsg) {
if (! JsonHelper::isArray(collection)) {
@ -1626,20 +1683,24 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
// we don't care about deleted collections
return TRI_ERROR_NO_ERROR;
}
TRI_vocbase_col_t* col = 0;
if (reuseId) {
TRI_json_t const* idString = JsonHelper::getArrayElement(parameters, "cid");
if (! JsonHelper::isString(idString)) {
errorMsg = "collection id is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
TRI_json_t const* idString = JsonHelper::getArrayElement(parameters, "cid");
if (! JsonHelper::isString(idString)) {
errorMsg = "collection id is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
TRI_voc_cid_t cid = StringUtils::uint64(idString->_value._string.data, idString->_value._string.length - 1);
// first look up the collection by the cid
col = TRI_LookupCollectionByIdVocBase(_vocbase, cid);
}
TRI_voc_cid_t cid = StringUtils::uint64(idString->_value._string.data, idString->_value._string.length - 1);
// first look up the collection by the cid
TRI_vocbase_col_t* col = TRI_LookupCollectionByIdVocBase(_vocbase, cid);
if (col == 0) {
// not found, try name next
@ -1667,7 +1728,7 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
}
// now re-create the collection
int res = createCollection(parameters, &col, remoteServerId);
int res = createCollection(parameters, &col, reuseId, remoteServerId);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "unable to create collection: " + string(TRI_errno_string(res));
@ -1678,6 +1739,107 @@ int RestReplicationHandler::processRestoreCollection (TRI_json_t* const collecti
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the indexes of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreIndexes (TRI_json_t* const collection,
TRI_server_id_t remoteServerId,
string& errorMsg) {
if (! JsonHelper::isArray(collection)) {
errorMsg = "collection declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(collection, "parameters");
if (! JsonHelper::isArray(parameters)) {
errorMsg = "collection parameters declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
TRI_json_t const* indexes = JsonHelper::getArrayElement(collection, "indexes");
if (! JsonHelper::isList(indexes)) {
errorMsg = "collection indexes declaration is invalid";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
const size_t n = indexes->_value._objects._length;
if (n == 0) {
// nothing to do
return TRI_ERROR_NO_ERROR;
}
const string name = JsonHelper::getStringValue(parameters, "name", "");
if (name.empty()) {
errorMsg = "collection name is missing";
return TRI_ERROR_HTTP_BAD_PARAMETER;
}
if (JsonHelper::getBooleanValue(parameters, "deleted", false)) {
// we don't care about deleted collections
return TRI_ERROR_NO_ERROR;
}
// look up the collection
TRI_vocbase_col_t* col = TRI_LookupCollectionByNameVocBase(_vocbase, name.c_str());
if (col == 0) {
errorMsg = "could not find collection '" + name + "'";
return TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND;
}
int res = TRI_UseCollectionVocBase(_vocbase, col);
if (res != TRI_ERROR_NO_ERROR) {
return res;
}
TRI_primary_collection_t* primary = col->_collection;
TRI_ReadLockReadWriteLock(&_vocbase->_inventoryLock);
TRI_WRITE_LOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* idxDef = (TRI_json_t const*) TRI_AtVector(&indexes->_value._objects, i);
TRI_index_t* idx = 0;
// {"id":"229907440927234","type":"hash","unique":false,"fields":["x","Y"]}
res = TRI_FromJsonIndexDocumentCollection((TRI_document_collection_t*) primary, idxDef, &idx);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "could not create index: " + string(TRI_errno_string(res));
break;
}
else {
assert(idx != 0);
res = TRI_SaveIndex(primary, idx, remoteServerId);
if (res != TRI_ERROR_NO_ERROR) {
errorMsg = "could not save index: " + string(TRI_errno_string(res));
break;
}
}
}
TRI_WRITE_UNLOCK_DOCUMENTS_INDEXES_PRIMARY_COLLECTION(primary);
TRI_ReadUnlockReadWriteLock(&_vocbase->_inventoryLock);
TRI_ReleaseCollectionVocBase(_vocbase, col);
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief apply the data from a collection dump or the continuous log
////////////////////////////////////////////////////////////////////////////////
@ -1721,7 +1883,7 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co
// parse _from
TRI_document_edge_t edge;
if (! DocumentHelper::parseDocumentId(_resolver, from.c_str(), edge._fromCid, &edge._fromKey)) {
if (! DocumentHelper::parseDocumentId(resolver, from.c_str(), edge._fromCid, &edge._fromKey)) {
res = TRI_ERROR_ARANGO_DOCUMENT_HANDLE_BAD;
}
@ -1797,14 +1959,14 @@ int RestReplicationHandler::applyCollectionDumpMarker (CollectionNameResolver co
/// @brief restores the data of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreDataBatch (TRI_transaction_collection_t* trxCollection,
int RestReplicationHandler::processRestoreDataBatch (CollectionNameResolver const& resolver,
TRI_transaction_collection_t* trxCollection,
TRI_server_id_t generatingServer,
bool useRevision,
std::string& errorMsg) {
const string invalidMsg = "received invalid JSON data for collection " +
StringUtils::itoa(trxCollection->_cid);
CollectionNameResolver resolver(_vocbase);
char const* ptr = _request->body();
char const* end = ptr + _request->bodySize();
@ -1864,7 +2026,7 @@ int RestReplicationHandler::processRestoreDataBatch (TRI_transaction_collection_
}
}
else if (TRI_EqualString(attributeName, "rev")) {
else if (useRevision && TRI_EqualString(attributeName, "rev")) {
if (JsonHelper::isString(value)) {
rid = StringUtils::uint64(value->_value._string.data, value->_value._string.length - 1);
}
@ -1904,8 +2066,10 @@ int RestReplicationHandler::processRestoreDataBatch (TRI_transaction_collection_
/// @brief restores the data of a collection TODO
////////////////////////////////////////////////////////////////////////////////
int RestReplicationHandler::processRestoreData (TRI_voc_cid_t cid,
int RestReplicationHandler::processRestoreData (CollectionNameResolver const& resolver,
TRI_voc_cid_t cid,
TRI_server_id_t generatingServer,
bool useRevision,
string& errorMsg) {
TRI_transaction_t* trx = TRI_CreateTransaction(_vocbase,
@ -1949,7 +2113,7 @@ int RestReplicationHandler::processRestoreData (TRI_voc_cid_t cid,
// TODO: waitForSync disabled here. use for initial replication, too
// sync at end of trx
trxCollection->_waitForSync = false;
res = processRestoreDataBatch(trxCollection, generatingServer, errorMsg);
res = processRestoreDataBatch(resolver, trxCollection, generatingServer, useRevision, errorMsg);
}
if (res == TRI_ERROR_NO_ERROR) {
@ -1966,16 +2130,18 @@ int RestReplicationHandler::processRestoreData (TRI_voc_cid_t cid,
////////////////////////////////////////////////////////////////////////////////
void RestReplicationHandler::handleCommandRestoreData () {
char const* collection = _request->value("collection");
char const* value = _request->value("collection");
if (collection == 0) {
if (value == 0) {
generateError(HttpResponse::BAD,
TRI_ERROR_HTTP_BAD_PARAMETER,
"invalid collection parameter");
return;
}
TRI_voc_cid_t cid = StringUtils::uint64(collection);
CollectionNameResolver resolver(_vocbase);
TRI_voc_cid_t cid = resolver.getCollectionId(value);
if (cid == 0) {
generateError(HttpResponse::BAD,
@ -1983,11 +2149,17 @@ void RestReplicationHandler::handleCommandRestoreData () {
"invalid collection parameter");
return;
}
bool recycleIds = false;
value = _request->value("recycleIds");
if (value != 0) {
recycleIds = StringUtils::boolean(value);
}
TRI_server_id_t remoteServerId = 0; // TODO
string errorMsg;
int res = processRestoreData(cid, remoteServerId, errorMsg);
int res = processRestoreData(resolver, cid, remoteServerId, recycleIds, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);

28
arangod/RestHandler/RestReplicationHandler.h
View File

@ -227,22 +227,38 @@ namespace triagens {
int createCollection (struct TRI_json_s const*,
struct TRI_vocbase_col_s**,
bool,
TRI_server_id_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief handle a restore command for a specific collection
////////////////////////////////////////////////////////////////////////////////
void handleCommandRestoreCollection ();
////////////////////////////////////////////////////////////////////////////////
/// @brief handle a restore command for a specific collection
////////////////////////////////////////////////////////////////////////////////
void handleCommandRestoreIndexes ();
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the structure of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int processRestoreCollection (struct TRI_json_s* const,
bool,
bool,
TRI_server_id_t,
std::string&);
////////////////////////////////////////////////////////////////////////////////
/// @brief handle a restore command for a specific collection
/// @brief restores the indexes of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
void handleCommandRestoreCollection ();
int processRestoreIndexes (struct TRI_json_s* const,
TRI_server_id_t,
std::string&);
////////////////////////////////////////////////////////////////////////////////
/// @brief apply a single marker from the collection dump
@ -260,16 +276,20 @@ namespace triagens {
/// @brief restores the data of a collection TODO MOVE
////////////////////////////////////////////////////////////////////////////////
int processRestoreDataBatch (struct TRI_transaction_collection_s*,
int processRestoreDataBatch (CollectionNameResolver const&,
struct TRI_transaction_collection_s*,
TRI_server_id_t,
bool,
std::string&);
////////////////////////////////////////////////////////////////////////////////
/// @brief restores the data of a collection TODO
////////////////////////////////////////////////////////////////////////////////
int processRestoreData (TRI_voc_cid_t,
int processRestoreData (CollectionNameResolver const&,
TRI_voc_cid_t,
TRI_server_id_t,
bool,
std::string&);
////////////////////////////////////////////////////////////////////////////////

25
arangosh/V8Client/arangodump.cpp
View File

@ -346,6 +346,13 @@ static string GetVersion () {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
}
else {
if (response->wasHttpError()) {
Client->setErrorMessage(GetHttpErrorMessage(response), false);
}
Connection->disconnect();
}
delete response;
@ -864,14 +871,6 @@ int main (int argc, char* argv[]) {
cerr << "output directory '" << OutputDirectory << "' already exists. use --overwrite to overwrite data in in it" << endl;
TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
}
else if (! isDirectory) {
int res = TRI_CreateDirectory(OutputDirectory.c_str());
if (res != TRI_ERROR_NO_ERROR) {
cerr << "unable to create output directory '" << OutputDirectory << "': " << string(TRI_errno_string(res)) << endl;
TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
}
}
// .............................................................................
// set-up client connection
@ -930,6 +929,16 @@ int main (int argc, char* argv[]) {
cerr << "Got an incompatible server version '" << versionString << "'" << endl;
TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
}
if (! isDirectory) {
int res = TRI_CreateDirectory(OutputDirectory.c_str());
if (res != TRI_ERROR_NO_ERROR) {
cerr << "unable to create output directory '" << OutputDirectory << "': " << string(TRI_errno_string(res)) << endl;
TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
}
}
if (Progress) {

232
arangosh/V8Client/arangorestore.cpp
View File

@ -131,6 +131,12 @@ static bool Progress = true;
static bool Overwrite = true;
////////////////////////////////////////////////////////////////////////////////
/// @brief re-use revision ids on import
////////////////////////////////////////////////////////////////////////////////
static bool RecycleIds = false;
////////////////////////////////////////////////////////////////////////////////
/// @}
////////////////////////////////////////////////////////////////////////////////
@ -155,6 +161,7 @@ static void ParseProgramOptions (int argc, char* argv[]) {
("collection", &Collections, "restrict to collection name (can be specified multiple times)")
("batch-size", &ChunkSize, "maximum size for individual data batches (in bytes)")
("import-data", &ImportData, "import data into collection")
("recycle-ids", &RecycleIds, "recycle collection and revision ids from dump")
("create-collection", &ImportStructure, "create collection structure")
("include-system-collections", &IncludeSystemCollections, "include system collections")
("input-directory", &InputDirectory, "input directory")
@ -326,6 +333,13 @@ static string GetVersion () {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
}
}
else {
if (response->wasHttpError()) {
Client->setErrorMessage(GetHttpErrorMessage(response), false);
}
Connection->disconnect();
}
delete response;
@ -333,15 +347,16 @@ static string GetVersion () {
}
////////////////////////////////////////////////////////////////////////////////
/// @brief send the request to re-create a single collection
/// @brief send the request to re-create a collection
////////////////////////////////////////////////////////////////////////////////
static int SendRestoreCollection (TRI_json_t const* json,
string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/restore-collection?overwrite=" +
string(Overwrite ? "true" : "false");
const string url = "/_api/replication/restore-collection"
"?overwrite=" + string(Overwrite ? "true" : "false") +
"&recycleIds=" + string(RecycleIds ? "true" : "false");
const string body = JsonHelper::toString(json);
@ -374,20 +389,63 @@ static int SendRestoreCollection (TRI_json_t const* json,
}
////////////////////////////////////////////////////////////////////////////////
/// @brief send the request to re-create data for a collection
/// @brief send the request to re-create indexes for a collection
////////////////////////////////////////////////////////////////////////////////
static int SendRestoreData (string const& cid,
string const& buffer,
string& errorMsg) {
static int SendRestoreIndexes (TRI_json_t const* json,
string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/restore-data?collection=" + cid;
const string url = "/_api/replication/restore-indexes";
const string body = JsonHelper::toString(json);
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
buffer.c_str(),
buffer.size(),
body.c_str(),
body.size(),
headers);
if (response == 0 || ! response->isComplete()) {
errorMsg = "got invalid response from server: " + Client->getErrorMessage();
if (response != 0) {
delete response;
}
return TRI_ERROR_INTERNAL;
}
if (response->wasHttpError()) {
errorMsg = GetHttpErrorMessage(response);
delete response;
return TRI_ERROR_INTERNAL;
}
delete response;
return TRI_ERROR_NO_ERROR;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief send the request to load data into a collection
////////////////////////////////////////////////////////////////////////////////
static int SendRestoreData (string const& cid,
string const& cname,
char const* buffer,
size_t bufferSize,
string& errorMsg) {
map<string, string> headers;
const string url = "/_api/replication/restore-data?collection=" +
StringUtils::urlEncode(cname) +
"&recycleIds=" + (RecycleIds ? "true" : "false");
SimpleHttpResult* response = Client->request(HttpRequest::HTTP_REQUEST_PUT,
url,
buffer,
bufferSize,
headers);
@ -442,6 +500,7 @@ static int SortCollections (const void* l,
////////////////////////////////////////////////////////////////////////////////
static int ProcessInputDirectory (string& errorMsg) {
// create a lookup table for collections
map<string, bool> restrictList;
for (size_t i = 0; i < Collections.size(); ++i) {
@ -455,7 +514,6 @@ static int ProcessInputDirectory (string& errorMsg) {
return TRI_ERROR_OUT_OF_MEMORY;
}
// step1: determine all collections to process
{
const vector<string> files = FileUtils::listFiles(InputDirectory);
@ -491,8 +549,12 @@ static int ProcessInputDirectory (string& errorMsg) {
const string fqn = InputDirectory + TRI_DIR_SEPARATOR_STR + files[i];
TRI_json_t* json = TRI_JsonFile(TRI_UNKNOWN_MEM_ZONE, fqn.c_str(), 0);
TRI_json_t const* parameters = JsonHelper::getArrayElement(json, "parameters");
TRI_json_t const* indexes = JsonHelper::getArrayElement(json, "indexes");
if (! JsonHelper::isArray(json)) {
if (! JsonHelper::isArray(json) ||
! JsonHelper::isArray(parameters) ||
! JsonHelper::isList(indexes)) {
errorMsg = "could not read collection structure file '" + name + "'";
if (json != 0) {
@ -504,22 +566,14 @@ static int ProcessInputDirectory (string& errorMsg) {
return TRI_ERROR_INTERNAL;
}
TRI_json_t const* parameters = JsonHelper::getArrayElement(json, "parameters");
if (! JsonHelper::isArray(parameters)) {
errorMsg = "invalid collection structure file '" + name + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
return TRI_ERROR_INTERNAL;
}
const string cname = JsonHelper::getStringValue(parameters, "name", "");
if (cname != name) {
errorMsg = "collection name mismatch in collection structure file '" + name + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, json);
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return TRI_ERROR_INTERNAL;
return TRI_ERROR_INTERNAL;
}
TRI_PushBack3ListJson(TRI_UNKNOWN_MEM_ZONE, collections, json);
@ -529,7 +583,8 @@ static int ProcessInputDirectory (string& errorMsg) {
// sort collections according to type (documents before edges)
qsort(collections->_value._objects._buffer, collections->_value._objects._length, sizeof(TRI_json_t), &SortCollections);
StringBuffer buffer(TRI_UNKNOWN_MEM_ZONE);
// step2: run the actual import
{
@ -537,12 +592,12 @@ static int ProcessInputDirectory (string& errorMsg) {
for (size_t i = 0; i < n; ++i) {
TRI_json_t const* json = (TRI_json_t const*) TRI_AtVector(&collections->_value._objects, i);
TRI_json_t const* parameters = JsonHelper::getArrayElement(json, "parameters");
TRI_json_t const* indexes = JsonHelper::getArrayElement(json, "indexes");
const string cname = JsonHelper::getStringValue(parameters, "name", "");
const string cid = JsonHelper::getStringValue(parameters, "cid", "");
if (ImportStructure) {
// re-create collection
if (Progress) {
cout << "Creating collection '" << cname << "'..." << endl;
}
@ -556,6 +611,7 @@ static int ProcessInputDirectory (string& errorMsg) {
}
}
if (ImportData) {
// import data. check if we have a datafile
// TODO: externalise file extension
@ -568,54 +624,110 @@ static int ProcessInputDirectory (string& errorMsg) {
cout << "Loading data into collection '" << cname << "'..." << endl;
}
ifstream ifs;
ifs.open(datafile.c_str(), ifstream::in | ifstream::binary);
if (! ifs.is_open()) {
int fd = TRI_OPEN(datafile.c_str(), O_RDONLY);
if (fd < 0) {
errorMsg = "cannot open collection data file '" + datafile + "'";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return TRI_ERROR_INTERNAL;
}
string buffer;
uint64_t totalLength = 0;
while (ifs.good()) {
string line;
buffer.clear();
std::getline(ifs, line, '\n');
size_t length = line.size() + 1;
if (line.size() > 2) {
buffer += line + '\n';
totalLength += (uint64_t) length;
if (totalLength > ChunkSize) {
int res = SendRestoreData(cid, buffer, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
ifs.close();
return res;
}
totalLength = 0;
buffer.clear();
}
while (true) {
if (buffer.reserve(16384) != TRI_ERROR_NO_ERROR) {
TRI_CLOSE(fd);
errorMsg = "out of memory";
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return TRI_ERROR_OUT_OF_MEMORY;
}
}
if (! buffer.empty()) {
int res = SendRestoreData(cid, buffer, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
ifs.close();
ssize_t numRead = TRI_READ(fd, buffer.end(), 16384);
if (numRead < 0) {
// error while reading
int res = TRI_errno();
TRI_CLOSE(fd);
errorMsg = string(TRI_errno_string(res));
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return res;
}
// read something
buffer.increaseLength(numRead);
if (buffer.length() < ChunkSize && numRead > 0) {
// still continue reading
continue;
}
// do we have a buffer?
if (buffer.length() > 0) {
// look for the last \n in the buffer
char* found = (char*) memrchr((const void*) buffer.begin(), '\n', buffer.length());
size_t length;
if (found == 0) {
// no \n found...
if (numRead == 0) {
// we're at the end. send the complete buffer anyway
length = buffer.length();
}
else {
// read more
continue;
}
}
else {
// found a \n somewhere
length = found - buffer.begin();
}
assert(length > 0);
int res = SendRestoreData(cid, cname, buffer.begin(), length, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_CLOSE(fd);
errorMsg = string(TRI_errno_string(res));
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return res;
}
buffer.erase_front(length);
}
if (numRead == 0) {
// EOF
break;
}
}
ifs.close();
TRI_CLOSE(fd);
}
}
if (ImportStructure) {
// re-create indexes
if (TRI_LengthVector(&indexes->_value._objects) > 0) {
// we actually have indexes
if (Progress) {
cout << "Creating indexes for collection '" << cname << "'..." << endl;
}
int res = SendRestoreIndexes(json, errorMsg);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeJson(TRI_UNKNOWN_MEM_ZONE, collections);
return TRI_ERROR_INTERNAL;
}
}
}
}
@ -722,7 +834,7 @@ int main (int argc, char* argv[]) {
cerr << "Error message: '" << Client->getErrorMessage() << "'" << endl;
TRI_EXIT_FUNCTION(EXIT_FAILURE, NULL);
}
// successfully connected
// validate server version

4
lib/Basics/StringUtils.cpp
View File

@ -1565,7 +1565,7 @@ namespace triagens {
string urlEncode (string const& str) {
return urlEncode(str.c_str(),str.size());
return urlEncode(str.c_str(), str.size());
}
@ -1573,7 +1573,7 @@ namespace triagens {
string urlEncode (const char* src) {
if (src != 0) {
size_t len = strlen(src);
return urlEncode(src,len);
return urlEncode(src, len);
}
return "";
}

2
lib/BasicsC/files.c
View File

@ -794,11 +794,9 @@ bool TRI_ReadPointer (int fd, void* buffer, size_t length) {
ptr = buffer;
while (0 < length) {
ssize_t n = TRI_READ(fd, ptr, (unsigned int) length);
if (n < 0) {
TRI_set_errno(TRI_ERROR_SYS_ERROR);
LOG_ERROR("cannot read: %s", TRI_LAST_ERROR_STR);

11
lib/HttpServer/HttpCommTask.h
View File

@ -576,7 +576,16 @@ namespace triagens {
// not found
else if (authResult == HttpResponse::NOT_FOUND) {
HttpResponse response(HttpResponse::NOT_FOUND);
response.setContentType("application/json; charset=utf-8");
response.body().appendText("{\"error\":true,\"errorMessage\":\"")
.appendText(TRI_errno_string(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND))
.appendText("\",\"code\":")
.appendInteger((int) authResult)
.appendText(",\"errorNum\":")
.appendInteger(TRI_ERROR_ARANGO_DATABASE_NOT_FOUND)
.appendText("}");
this->handleResponse(&response);
this->resetState();
}

4
lib/SimpleHttpClient/SimpleHttpClient.h
View File

@ -137,8 +137,6 @@ namespace triagens {
return _errorMessage;
}
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief register and dump an error message
////////////////////////////////////////////////////////////////////////////////
@ -163,6 +161,8 @@ namespace triagens {
setErrorMessage(message);
}
}
private:
////////////////////////////////////////////////////////////////////////////////
/// @brief returns true if the request is in progress