Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel' of https://github.com/triAGENS/ArangoDB into devel

This commit is contained in:
scottashton 2014-06-23 14:42:41 +02:00
parent da1cb2e62d 73cecaeef6
commit 5e2fa60521
17 changed files with 360 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Documentation/Books/Users/General-Graphs/Management.mdpp
View File

@ -50,7 +50,6 @@ After having introduced edge definitions and orphan collections a graph can be c
@startDocuBlock JSF_general_graph_create
!SUBSUBSECTION Complete Example to create a graph
Example Call:

141
Documentation/Books/Users/General-Graphs/Management.orig.mdpp
View File

@ -1,141 +0,0 @@
!CHAPTER Graph Management
In order to create a graph the philosophy of handling the graph content has to introduced.
A graph contains a set of edge definitions each referring to one edge collection and
defining constraints on the vertex collections used as start and end points of the edges.
Furthermore a graph can contain an arbitrary amount of vertex collections, called orphan collections, that are not used in any edge definition but should be managed by the graph.
In order to create a graph the functionality to create edge definitions has to be introduced first:
!SECTION Edge Definitions
The edge definitions for a graph is an Array containing arbitrary many directed and/or undirected relations as defined below.
!SUBSECTION Initialize the list
<!-- @startDocuBlock JSF_general_graph_edge_definitions -->
!SUBSECTION Extend the list
<!-- @startDocuBlock JSF_general_graph_extend_edge_definitions -->
!SUBSUBSECTION Undirected Relation
<!-- @startDocuBlock JSF_general_graph_undirectedRelation -->
!SUBSUBSECTION Directed Relation
<!-- @startDocuBlock JSF_general_graph_directedRelation -->
!SUBSECTION Orphan Collections
Each graph has an orphan collection. It consists of arbitrary many vertex collection (type *document*), that are not
used in an edge definition of the graph. If the graph is extended with an edge definition using one of the orphans,
it will be removed from the orphan collection automatically.
!SUBSUBSECTION Add
<!-- @startDocuBlock JSF_general_graph__addOrphanCollection -->
!SUBSUBSECTION Read
<!-- @startDocuBlock JSF_general_graph__getOrphanCollections -->
!SUBSUBSECTION Remove
<!-- @startDocuBlock JSF_general_graph__removeOrphanCollection -->
!SECTION Create a graph
After having introduced edge definitions and orphan collections a graph can be created.
<!-- @startDocuBlock JSF_general_graph_create -->
!SUBSUBSECTION Complete Example to create a graph
Example Call:
```js
> var graph = require("org/arangodb/graph");
> var edgeDefinitions = graph._edgeDefinitions();
> graph._extendEdgeDefinitions(edgeDefinitions, graph._undirectedRelation("friend_of", ["Customer"]));
> graph._extendEdgeDefinitions(edgeDefinitions, graph._directedRelation("has_bought", ["Customer", "Company"], ["Groceries", "Electronics"]));
> graph._create("myStore", edgeDefinitions);
{
_id: "_graphs/123",
_rev: "123",
_key: "123"
}
```
alternative call:
```js
> var graph = require("org/arangodb/graph");
> var edgeDefinitions = graph._edgeDefinitions(graph._undirectedRelation("friend_of", ["Customer"]), graph._directedRelation("has_bought", ["Customer", "Company"], ["Groceries", "Electronics"]));
> graph._create("myStore", edgeDefinitions);
{
_id: "_graphs/123",
_rev: "123",
_key: "123"
};
```
!SUBSECTION List available graphs
<!-- @startDocuBlock JSF_general_graph_list_call -->
`general-graph._list()` *List all graphs.*
<br />
<br />
<!-- @startDocuBlock JSF_general_graph_list_info -->
<br />
@EXAMPLES
<br />
<!-- @startDocuBlock JSF_general_graph_list_examples -->
!SUBSECTION Load a graph
<!-- @startDocuBlock JSF_general_graph_graph -->
!SUBSECTION Remove a graph
<!-- @startDocuBlock JSF_general_graph_drop -->
!SECTION Edge
!SUBSECTION Save
<!-- @startDocuBlock JSF_general_graph_vertex_collection_save -->
!SUBSECTION Replace
<!-- @startDocuBlock JSF_general_graph_vertex_collection_replace -->
!SUBSECTION Update
<!-- @startDocuBlock JSF_general_graph_vertex_collection_update -->
!SUBSECTION Remove
<!-- @startDocuBlock JSF_general_graph_vertex_collection_remove -->
!SECTION Edge
!SUBSECTION Save
<!-- @startDocuBlock JSF_general_graph_edge_collection_save -->
!SUBSECTION Replace
<!-- @startDocuBlock JSF_general_graph_edge_collection_replace -->
!SUBSECTION Update
<!-- @startDocuBlock JSF_general_graph_edge_collection_update -->
!SUBSECTION Remove
<!-- @startDocuBlock JSF_general_graph_edge_collection_remove -->

49
Documentation/Books/codeBlockReader.py
View File

@ -31,15 +31,20 @@ def example_content(filepath, fh, tag):
""" Fetches an example file and inserts it using code
"""
first = True
arangosh = False
showdots = True
curl = False
first = True
lastline = None
short = ""
shortLines = 0
long = ""
longLines = 0
short = ""
shortLines = 0
shortable = False
showdots = True
CURL_STATE_CMD = 1
CURL_STATE_HEADER = 2
CURL_STATE_BODY = 3
# read in the context, split into long and short
infile = open(filepath, 'r')
@ -47,6 +52,7 @@ def example_content(filepath, fh, tag):
for line in infile:
if first:
arangosh = line.startswith("arangosh>")
curl = line.startswith("shell> curl")
first = False
if arangosh:
@ -57,11 +63,15 @@ def example_content(filepath, fh, tag):
lastline = None
short = short + line
shortLines = shortLines + 1
showdots = True
else:
if showdots:
if lastline == None:
lastline = line
# lastline = line
shortable = True
showdots = False
lastline = None
else:
# short = short + "~~~hidden~~~\n"
# shortLines = shortLines + 1
@ -69,12 +79,26 @@ def example_content(filepath, fh, tag):
showdots = False
lastline = None
if curl:
if line.startswith("shell> curl"):
curlState = CURL_STATE_CMD
elif curlState == CURL_STATE_CMD and line.startswith("HTTP/1.1 "):
curlState = CURL_STATE_HEADER
elif curlState == CURL_STATE_HEADER and line.startswith("{"):
curlState = CURL_STATE_BODY
if curlState == CURL_STATE_CMD or curlState == CURL_STATE_HEADER:
short = short + line
shortLines = shortLines + 1
else:
shortable = True
long = long + line
longLines = longLines + 1
# if lastline != None:
# short = short + lastline
# shortLines = shortLines + 1
if lastline != None:
short = short + lastline
shortLines = shortLines + 1
infile.close()
@ -109,7 +133,14 @@ def example_content(filepath, fh, tag):
fh.write("```\n")
fh.write("%s" % short)
fh.write("```\n")
fh.write("</pre><div class=\"example_show_button\">show execution results</div>\n")
if arangosh:
fh.write("</pre><div class=\"example_show_button\">show execution results</div>\n")
elif curl:
fh.write("</pre><div class=\"example_show_button\">show response body</div>\n")
else:
fh.write("</pre><div class=\"example_show_button\">show</div>\n")
fh.write("</div>\n")
fh.write("</div>\n")

1
Documentation/Makefile.files
View File

@ -350,6 +350,7 @@ examples:
@srcdir@/js/client \
@srcdir@/js/common \
@srcdir@/js/server \
@srcdir@/js/apps/system/gharial \
@srcdir@/Documentation/Books/Users \
@srcdir@/arangod/RestHandler \
@srcdir@/lib/Admin \

126
arangod/RestHandler/RestReplicationHandler.cpp
View File

@ -1331,64 +1331,68 @@ void RestReplicationHandler::handleCommandLoggerFollow () {
return;
}
uint64_t const chunkSize = determineChunkSize();
int res = TRI_ERROR_NO_ERROR;
// initialise the dump container
TRI_replication_dump_t dump;
if (TRI_InitDumpReplication(&dump, _vocbase, (size_t) defaultChunkSize) != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, TRI_ERROR_OUT_OF_MEMORY);
return;
try {
// initialise the dump container
TRI_replication_dump_t dump(_vocbase, (size_t) determineChunkSize());
// and dump
res = TRI_DumpLogReplication(_vocbase, &dump, tickStart, tickEnd);
if (res == TRI_ERROR_NO_ERROR) {
bool const checkMore = (dump._lastFoundTick > 0 && dump._lastFoundTick != state.lastDataTick);
// generate the result
size_t const length = TRI_LengthStringBuffer(dump._buffer);
if (length == 0) {
_response = createResponse(HttpResponse::NO_CONTENT);
}
else {
_response = createResponse(HttpResponse::OK);
}
_response->setContentType("application/x-arango-dump; charset=utf-8");
// set headers
_response->setHeader(TRI_REPLICATION_HEADER_CHECKMORE,
strlen(TRI_REPLICATION_HEADER_CHECKMORE),
checkMore ? "true" : "false");
_response->setHeader(TRI_REPLICATION_HEADER_LASTINCLUDED,
strlen(TRI_REPLICATION_HEADER_LASTINCLUDED),
StringUtils::itoa(dump._lastFoundTick));
_response->setHeader(TRI_REPLICATION_HEADER_LASTTICK,
strlen(TRI_REPLICATION_HEADER_LASTTICK),
StringUtils::itoa(state.lastTick));
_response->setHeader(TRI_REPLICATION_HEADER_ACTIVE,
strlen(TRI_REPLICATION_HEADER_ACTIVE),
"true");
if (length > 0) {
// transfer ownership of the buffer contents
_response->body().set(dump._buffer);
// to avoid double freeing
TRI_StealStringBuffer(dump._buffer);
}
insertClient(dump._lastFoundTick);
}
}
catch (triagens::arango::Exception const& ex) {
res = ex.code();
}
catch (...) {
res = TRI_ERROR_INTERNAL;
}
int res = TRI_DumpLogReplication(_vocbase, &dump, tickStart, tickEnd, chunkSize);
if (res == TRI_ERROR_NO_ERROR) {
bool const checkMore = (dump._lastFoundTick > 0 && dump._lastFoundTick != state.lastDataTick);
// generate the result
size_t const length = TRI_LengthStringBuffer(dump._buffer);
if (length == 0) {
_response = createResponse(HttpResponse::NO_CONTENT);
}
else {
_response = createResponse(HttpResponse::OK);
}
_response->setContentType("application/x-arango-dump; charset=utf-8");
// set headers
_response->setHeader(TRI_REPLICATION_HEADER_CHECKMORE,
strlen(TRI_REPLICATION_HEADER_CHECKMORE),
checkMore ? "true" : "false");
_response->setHeader(TRI_REPLICATION_HEADER_LASTINCLUDED,
strlen(TRI_REPLICATION_HEADER_LASTINCLUDED),
StringUtils::itoa(dump._lastFoundTick));
_response->setHeader(TRI_REPLICATION_HEADER_LASTTICK,
strlen(TRI_REPLICATION_HEADER_LASTTICK),
StringUtils::itoa(state.lastTick));
_response->setHeader(TRI_REPLICATION_HEADER_ACTIVE,
strlen(TRI_REPLICATION_HEADER_ACTIVE),
"true");
if (length > 0) {
// transfer ownership of the buffer contents
_response->body().set(dump._buffer);
// avoid double freeing
TRI_StealStringBuffer(dump._buffer);
}
insertClient(dump._lastFoundTick);
}
else {
if (res != TRI_ERROR_NO_ERROR) {
generateError(HttpResponse::SERVER_ERROR, res);
}
TRI_DestroyDumpReplication(&dump);
}
////////////////////////////////////////////////////////////////////////////////
@ -3177,8 +3181,6 @@ void RestReplicationHandler::handleCommandDump () {
translateCollectionIds = StringUtils::boolean(value);
}
uint64_t const chunkSize = determineChunkSize();
TRI_vocbase_col_t* c = TRI_LookupCollectionByNameVocBase(_vocbase, collection);
if (c == nullptr) {
@ -3204,19 +3206,13 @@ void RestReplicationHandler::handleCommandDump () {
TRI_ASSERT(col != nullptr);
// initialise the dump container
TRI_replication_dump_t dump;
res = TRI_InitDumpReplication(&dump, _vocbase, (size_t) defaultChunkSize);
TRI_replication_dump_t dump(_vocbase, (size_t) determineChunkSize());
res = TRI_DumpCollectionReplication(&dump, col, tickStart, tickEnd, withTicks, translateCollectionIds);
if (res != TRI_ERROR_NO_ERROR) {
THROW_ARANGO_EXCEPTION(res);
}
res = TRI_DumpCollectionReplication(&dump, col, tickStart, tickEnd, chunkSize, withTicks, translateCollectionIds);
if (res != TRI_ERROR_NO_ERROR) {
TRI_DestroyDumpReplication(&dump);
THROW_ARANGO_EXCEPTION(res);
}
// generate the result
size_t const length = TRI_LengthStringBuffer(dump._buffer);
@ -3244,8 +3240,6 @@ void RestReplicationHandler::handleCommandDump () {
// avoid double freeing
TRI_StealStringBuffer(dump._buffer);
TRI_DestroyDumpReplication(&dump);
}
catch (triagens::arango::Exception const& ex) {
res = ex.code();

6
arangod/VocBase/collection.h
View File

@ -100,6 +100,12 @@ struct TRI_vocbase_col_s;
#define TRI_COL_NAME_TRANSACTION "_trx"
////////////////////////////////////////////////////////////////////////////////
/// @brief predefined system collection name for replication
////////////////////////////////////////////////////////////////////////////////
#define TRI_COL_NAME_REPLICATION "_replication"
////////////////////////////////////////////////////////////////////////////////
/// @brief predefined collection name for users
////////////////////////////////////////////////////////////////////////////////

6
arangod/VocBase/replication-common.cpp
View File

@ -62,7 +62,7 @@ void TRI_GetTimeStampReplication (char* dst,
////////////////////////////////////////////////////////////////////////////////
bool TRI_ExcludeCollectionReplication (const char* name) {
if (name == NULL) {
if (name == nullptr) {
// name invalid
return true;
}
@ -72,7 +72,9 @@ bool TRI_ExcludeCollectionReplication (const char* name) {
return false;
}
if (TRI_EqualString(name, TRI_COL_NAME_USERS) ||
if (TRI_EqualString(name, TRI_COL_NAME_REPLICATION) ||
TRI_EqualString(name, TRI_COL_NAME_TRANSACTION) ||
TRI_EqualString(name, TRI_COL_NAME_USERS) ||
TRI_IsPrefixString(name, TRI_COL_NAME_STATISTICS) ||
TRI_EqualString(name, "_aal") ||
TRI_EqualString(name, "_fishbowl") ||

367
arangod/VocBase/replication-dump.cpp
View File

@ -33,7 +33,6 @@
#include "BasicsC/files.h"
#include "BasicsC/json.h"
#include "BasicsC/logging.h"
#include "BasicsC/string-buffer.h"
#include "BasicsC/tri-strings.h"
#include "VocBase/collection.h"
@ -42,8 +41,6 @@
#include "VocBase/transaction.h"
#include "VocBase/vocbase.h"
#include "VocBase/voc-shaper.h"
#include "Utils/Exception.h"
#include "Utils/transactions.h"
#include "Wal/Logfile.h"
#include "Wal/LogfileManager.h"
#include "Wal/Marker.h"
@ -108,86 +105,36 @@ typedef struct df_entry_s {
}
df_entry_t;
////////////////////////////////////////////////////////////////////////////////
/// @brief container for a resolved collection name (cid => name)
////////////////////////////////////////////////////////////////////////////////
typedef struct resolved_name_s {
TRI_voc_cid_t _cid;
char* _name;
}
resolved_name_t;
// -----------------------------------------------------------------------------
// --SECTION-- private functions
// -----------------------------------------------------------------------------
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a collection id
/// @brief translate a collection id into a collection name
////////////////////////////////////////////////////////////////////////////////
char const* NameFromCid (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid) {
auto it = dump->_collectionNames.find(cid);
static uint64_t HashKeyCid (TRI_associative_pointer_t* array,
void const* key) {
TRI_voc_cid_t const* k = static_cast<TRI_voc_cid_t const*>(key);
if (it != dump->_collectionNames.end()) {
// collection name is in cache already
return (*it).second.c_str();
}
// collection name not in cache yet
char* name = TRI_GetCollectionNameByIdVocBase(dump->_vocbase, cid);
return *k;
}
if (name != nullptr) {
// insert into cache
dump->_collectionNames.insert(it, std::make_pair(cid, std::string(name)));
TRI_FreeString(TRI_UNKNOWN_MEM_ZONE, name);
////////////////////////////////////////////////////////////////////////////////
/// @brief hashes a collection name
////////////////////////////////////////////////////////////////////////////////
static uint64_t HashElementCid (TRI_associative_pointer_t* array,
void const* element) {
resolved_name_t const* e = static_cast<resolved_name_t const*>(element);
return e->_cid;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief compares a collection
////////////////////////////////////////////////////////////////////////////////
static bool IsEqualKeyElementCid (TRI_associative_pointer_t* array,
void const* key,
void const* element) {
TRI_voc_cid_t const* k = static_cast<TRI_voc_cid_t const*>(key);
resolved_name_t const* e = static_cast<resolved_name_t const*>(element);
return *k == e->_cid;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief lookup a collection name
////////////////////////////////////////////////////////////////////////////////
static bool LookupCollectionName (TRI_replication_dump_t* dump,
TRI_voc_cid_t cid,
char** result) {
TRI_ASSERT(cid > 0);
resolved_name_t* found = static_cast<resolved_name_t*>(TRI_LookupByKeyAssociativePointer(&dump->_collectionNames, &cid));
if (found == NULL) {
found = static_cast<resolved_name_t*>(TRI_Allocate(TRI_UNKNOWN_MEM_ZONE, sizeof(resolved_name_t), false));
if (found == NULL) {
// out of memory;
return false;
}
found->_cid = cid;
// name can be NULL if collection is not found.
// but we will still cache a NULL result!
found->_name = TRI_GetCollectionNameByIdVocBase(dump->_vocbase, cid);
TRI_InsertKeyAssociativePointer(&dump->_collectionNames, &found->_cid, found, false);
// and look it up again
return NameFromCid(dump, cid);
}
*result = found->_name;
return true;
return nullptr;
}
////////////////////////////////////////////////////////////////////////////////
@ -199,13 +146,9 @@ static bool AppendCollection (TRI_replication_dump_t* dump,
bool translateCollectionIds) {
if (translateCollectionIds) {
if (cid > 0) {
char* name;
char const* name = NameFromCid(dump, cid);
if (! LookupCollectionName(dump, cid, &name)) {
return false;
}
if (name != NULL) {
if (name != nullptr) {
APPEND_STRING(dump->_buffer, name);
return true;
}
@ -306,6 +249,21 @@ static TRI_vector_t GetRangeDatafiles (TRI_document_collection_t* document,
return datafiles;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief append database id plus collection id
////////////////////////////////////////////////////////////////////////////////
static bool AppendContext (TRI_replication_dump_t* dump,
TRI_voc_tick_t databaseId,
TRI_voc_cid_t collectionId) {
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, collectionId);
APPEND_STRING(dump->_buffer, "\",");
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief stringify a raw marker from a datafile for a collection dump
////////////////////////////////////////////////////////////////////////////////
@ -487,12 +445,12 @@ static bool StringifyMarkerDump (TRI_replication_dump_t* dump,
static bool StringifyWalMarkerDocument (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::document_marker_t const*>(marker);
if (! AppendContext(dump, m->_databaseId, m->_collectionId)) {
return false;
}
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
APPEND_STRING(dump->_buffer, "\",\"tid\":\"");
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
@ -513,7 +471,7 @@ static bool StringifyWalMarkerDocument (TRI_replication_dump_t* dump,
shaped._data.data = (char*) m + m->_offsetJson;
triagens::basics::LegendReader lr((char const*) m + m->_offsetLegend);
if (! TRI_StringifyArrayShapedJson(&lr, dump->_buffer, &shaped, false)) {
if (! TRI_StringifyArrayShapedJson(&lr, dump->_buffer, &shaped, true)) {
return false;
}
@ -529,12 +487,12 @@ static bool StringifyWalMarkerDocument (TRI_replication_dump_t* dump,
static bool StringifyWalMarkerEdge (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::edge_marker_t const*>(marker);
if (! AppendContext(dump, m->_databaseId, m->_collectionId)) {
return false;
}
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
APPEND_STRING(dump->_buffer, "\",\"tid\":\"");
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetKey);
@ -550,13 +508,13 @@ static bool StringifyWalMarkerEdge (TRI_replication_dump_t* dump,
// from
APPEND_STRING(dump->_buffer, ",\"" TRI_VOC_ATTRIBUTE_FROM "\":\"");
APPEND_UINT64(dump->_buffer, m->_fromCid);
APPEND_UINT64(dump->_buffer, (uint64_t) m->_fromCid);
APPEND_STRING(dump->_buffer, "\\/");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetFromKey);
// to
APPEND_STRING(dump->_buffer, "\",\"" TRI_VOC_ATTRIBUTE_TO "\":\"");
APPEND_UINT64(dump->_buffer, m->_toCid);
APPEND_UINT64(dump->_buffer, (uint64_t) m->_toCid);
APPEND_STRING(dump->_buffer, "\\/");
APPEND_STRING(dump->_buffer, (char const*) m + m->_offsetFromKey);
APPEND_STRING(dump->_buffer, "\"");
@ -567,7 +525,7 @@ static bool StringifyWalMarkerEdge (TRI_replication_dump_t* dump,
shaped._data.data = (char*) m + m->_offsetJson;
triagens::basics::LegendReader lr((char const*) m + m->_offsetLegend);
if (! TRI_StringifyArrayShapedJson(&lr, dump->_buffer, &shaped, false)) {
if (! TRI_StringifyArrayShapedJson(&lr, dump->_buffer, &shaped, true)) {
return false;
}
@ -584,11 +542,10 @@ static bool StringifyWalMarkerRemove (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
auto m = reinterpret_cast<triagens::wal::remove_marker_t const*>(marker);
APPEND_STRING(dump->_buffer, "\"database\":\"");
APPEND_UINT64(dump->_buffer, m->_databaseId);
APPEND_STRING(dump->_buffer, "\",\"cid\":\"");
APPEND_UINT64(dump->_buffer, m->_collectionId);
APPEND_STRING(dump->_buffer, "\",\"tid\":\"");
if (! AppendContext(dump, m->_databaseId, m->_collectionId)) {
return false;
}
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
APPEND_STRING(dump->_buffer, (char const*) m + sizeof(triagens::wal::remove_marker_t));
@ -845,10 +802,108 @@ static bool StringifyWalMarker (TRI_replication_dump_t* dump,
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a marker is replicated
/// @brief helper function to extract a database id from a marker
////////////////////////////////////////////////////////////////////////////////
template<typename T>
static TRI_voc_tick_t GetDatabaseId (TRI_df_marker_t const* marker) {
T const* m = reinterpret_cast<T const*>(marker);
return m->_databaseId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the database id from a marker
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_tick_t GetDatabaseFromWalMarker (TRI_df_marker_t const* marker) {
switch (marker->_type) {
case TRI_WAL_MARKER_ATTRIBUTE:
return GetDatabaseId<triagens::wal::attribute_marker_t>(marker);
case TRI_WAL_MARKER_SHAPE:
return GetDatabaseId<triagens::wal::shape_marker_t>(marker);
case TRI_WAL_MARKER_DOCUMENT:
return GetDatabaseId<triagens::wal::document_marker_t>(marker);
case TRI_WAL_MARKER_EDGE:
return GetDatabaseId<triagens::wal::edge_marker_t>(marker);
case TRI_WAL_MARKER_REMOVE:
return GetDatabaseId<triagens::wal::remove_marker_t>(marker);
case TRI_WAL_MARKER_BEGIN_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_begin_marker_t>(marker);
case TRI_WAL_MARKER_COMMIT_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_commit_marker_t>(marker);
case TRI_WAL_MARKER_ABORT_TRANSACTION:
return GetDatabaseId<triagens::wal::transaction_abort_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_COLLECTION:
return GetDatabaseId<triagens::wal::collection_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_COLLECTION:
return GetDatabaseId<triagens::wal::collection_drop_marker_t>(marker);
case TRI_WAL_MARKER_RENAME_COLLECTION:
return GetDatabaseId<triagens::wal::collection_rename_marker_t>(marker);
case TRI_WAL_MARKER_CHANGE_COLLECTION:
return GetDatabaseId<triagens::wal::collection_change_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_INDEX:
return GetDatabaseId<triagens::wal::index_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_INDEX:
return GetDatabaseId<triagens::wal::index_drop_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_DATABASE:
return GetDatabaseId<triagens::wal::database_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_DATABASE:
return GetDatabaseId<triagens::wal::database_drop_marker_t>(marker);
default: {
return 0;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief helper function to extract a collection id from a marker
////////////////////////////////////////////////////////////////////////////////
template<typename T>
static TRI_voc_tick_t GetCollectionId (TRI_df_marker_t const* marker) {
T const* m = reinterpret_cast<T const*>(marker);
return m->_collectionId;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief get the collection id from a marker
////////////////////////////////////////////////////////////////////////////////
static TRI_voc_tick_t GetCollectionFromWalMarker (TRI_df_marker_t const* marker) {
switch (marker->_type) {
case TRI_WAL_MARKER_ATTRIBUTE:
return GetCollectionId<triagens::wal::attribute_marker_t>(marker);
case TRI_WAL_MARKER_SHAPE:
return GetCollectionId<triagens::wal::shape_marker_t>(marker);
case TRI_WAL_MARKER_DOCUMENT:
return GetCollectionId<triagens::wal::document_marker_t>(marker);
case TRI_WAL_MARKER_EDGE:
return GetCollectionId<triagens::wal::edge_marker_t>(marker);
case TRI_WAL_MARKER_REMOVE:
return GetCollectionId<triagens::wal::remove_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_COLLECTION:
return GetCollectionId<triagens::wal::collection_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_COLLECTION:
return GetCollectionId<triagens::wal::collection_drop_marker_t>(marker);
case TRI_WAL_MARKER_RENAME_COLLECTION:
return GetCollectionId<triagens::wal::collection_rename_marker_t>(marker);
case TRI_WAL_MARKER_CHANGE_COLLECTION:
return GetCollectionId<triagens::wal::collection_change_marker_t>(marker);
case TRI_WAL_MARKER_CREATE_INDEX:
return GetCollectionId<triagens::wal::index_create_marker_t>(marker);
case TRI_WAL_MARKER_DROP_INDEX:
return GetCollectionId<triagens::wal::index_drop_marker_t>(marker);
default: {
return 0;
}
}
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a marker should be replicated
////////////////////////////////////////////////////////////////////////////////
static inline bool MustReplicateMarker (TRI_df_marker_t const* marker) {
static inline bool MustReplicateWalMarkerType (TRI_df_marker_t const* marker) {
return (marker->_type == TRI_WAL_MARKER_DOCUMENT ||
marker->_type == TRI_WAL_MARKER_EDGE ||
marker->_type == TRI_WAL_MARKER_REMOVE ||
@ -863,6 +918,35 @@ static inline bool MustReplicateMarker (TRI_df_marker_t const* marker) {
marker->_type == TRI_WAL_MARKER_DROP_INDEX);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a marker is replicated
////////////////////////////////////////////////////////////////////////////////
static bool MustReplicateWalMarker (TRI_replication_dump_t* dump,
TRI_df_marker_t const* marker) {
// first check the marker type
if (! MustReplicateWalMarkerType(marker)) {
return false;
}
// then check if the marker belongs to the "correct" database
if (dump->_vocbase->_id != GetDatabaseFromWalMarker(marker)) {
return false;
}
// finally check if the marker is for a collection that we want to ignore
TRI_voc_cid_t cid = GetCollectionFromWalMarker(marker);
if (cid != 0) {
char const* name = NameFromCid(dump, cid);
if (name != nullptr && TRI_ExcludeCollectionReplication(name)) {
return false;
}
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief dump data from a collection
////////////////////////////////////////////////////////////////////////////////
@ -871,7 +955,6 @@ static int DumpCollection (TRI_replication_dump_t* dump,
TRI_document_collection_t* document,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize,
bool withTicks,
bool translateCollectionIds) {
TRI_vector_t datafiles;
@ -892,11 +975,10 @@ static int DumpCollection (TRI_replication_dump_t* dump,
// until a certain tick.
triagens::arango::TransactionBase trx(true);
LOG_TRACE("dumping collection %llu, tick range %llu - %llu, chunk size %llu",
LOG_TRACE("dumping collection %llu, tick range %llu - %llu",
(unsigned long long) document->_info._cid,
(unsigned long long) dataMin,
(unsigned long long) dataMax,
(unsigned long long) chunkSize);
(unsigned long long) dataMax);
buffer = dump->_buffer;
datafiles = GetRangeDatafiles(document, dataMin, dataMax);
@ -1034,7 +1116,7 @@ static int DumpCollection (TRI_replication_dump_t* dump,
goto NEXT_DF;
}
if ((uint64_t) TRI_LengthStringBuffer(buffer) > chunkSize) {
if ((uint64_t) TRI_LengthStringBuffer(buffer) > dump->_chunkSize) {
// abort the iteration
bufferFull = true;
@ -1085,7 +1167,6 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump,
TRI_vocbase_col_t* col,
TRI_voc_tick_t dataMin,
TRI_voc_tick_t dataMax,
uint64_t chunkSize,
bool withTicks,
bool translateCollectionIds) {
TRI_ASSERT(col != nullptr);
@ -1103,7 +1184,7 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump,
// block compaction
TRI_ReadLockReadWriteLock(&document->_compactionLock);
int res = DumpCollection(dump, document, dataMin, dataMax, chunkSize, withTicks, translateCollectionIds);
int res = DumpCollection(dump, document, dataMin, dataMax, withTicks, translateCollectionIds);
TRI_ReadUnlockReadWriteLock(&document->_compactionLock);
@ -1119,12 +1200,10 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t* dump,
int TRI_DumpLogReplication (TRI_vocbase_t* vocbase,
TRI_replication_dump_t* dump,
TRI_voc_tick_t tickMin,
TRI_voc_tick_t tickMax,
uint64_t chunkSize) {
LOG_TRACE("dumping log, tick range %llu - %llu, chunk size %llu",
TRI_voc_tick_t tickMax) {
LOG_TRACE("dumping log, tick range %llu - %llu",
(unsigned long long) tickMin,
(unsigned long long) tickMax,
(unsigned long long) chunkSize);
(unsigned long long) tickMax);
// ask the logfile manager which datafiles qualify
std::vector<triagens::wal::Logfile*> logfiles = triagens::wal::LogfileManager::instance()->getLogfilesForTickRange(tickMin, tickMax);
@ -1172,8 +1251,8 @@ int TRI_DumpLogReplication (TRI_vocbase_t* vocbase,
// marker too new
break;
}
if (! MustReplicateMarker(marker)) {
if (! MustReplicateWalMarker(dump, marker)) {
// check if we can abort searching
continue;
}
@ -1185,7 +1264,7 @@ int TRI_DumpLogReplication (TRI_vocbase_t* vocbase,
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
if ((uint64_t) TRI_LengthStringBuffer(dump->_buffer) > chunkSize) {
if ((uint64_t) TRI_LengthStringBuffer(dump->_buffer) > dump->_chunkSize) {
// abort the iteration
bufferFull = true;
break;
@ -1227,66 +1306,6 @@ int TRI_DumpLogReplication (TRI_vocbase_t* vocbase,
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief initialise a replication dump container
////////////////////////////////////////////////////////////////////////////////
int TRI_InitDumpReplication (TRI_replication_dump_t* dump,
TRI_vocbase_t* vocbase,
size_t bufferSize) {
int res;
TRI_ASSERT(vocbase != nullptr);
dump->_vocbase = vocbase;
dump->_lastFoundTick = 0;
dump->_lastSid = 0;
dump->_lastShape = nullptr;
dump->_failed = false;
dump->_bufferFull = false;
dump->_hasMore = false;
dump->_buffer = TRI_CreateSizedStringBuffer(TRI_CORE_MEM_ZONE, bufferSize);
if (dump->_buffer == nullptr) {
return TRI_ERROR_OUT_OF_MEMORY;
}
res = TRI_InitAssociativePointer(&dump->_collectionNames,
TRI_UNKNOWN_MEM_ZONE,
HashKeyCid,
HashElementCid,
IsEqualKeyElementCid,
nullptr);
if (res != TRI_ERROR_NO_ERROR) {
TRI_FreeStringBuffer(TRI_CORE_MEM_ZONE, dump->_buffer);
}
return res;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a replication dump container
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDumpReplication (TRI_replication_dump_t* dump) {
for (size_t i = 0; i < dump->_collectionNames._nrAlloc; ++i) {
resolved_name_t* found = static_cast<resolved_name_t*>(dump->_collectionNames._table[i]);
if (found != nullptr) {
if (found->_name != nullptr) {
// name can be NULL
TRI_Free(TRI_UNKNOWN_MEM_ZONE, found->_name);
}
TRI_Free(TRI_UNKNOWN_MEM_ZONE, found);
}
}
TRI_DestroyAssociativePointer(&dump->_collectionNames);
TRI_FreeStringBuffer(TRI_CORE_MEM_ZONE, dump->_buffer);
}
// -----------------------------------------------------------------------------
// --SECTION-- END-OF-FILE
// -----------------------------------------------------------------------------

62
arangod/VocBase/replication-dump.h
View File

@ -33,18 +33,19 @@
#include "Basics/Common.h"
#include "BasicsC/associative.h"
#include "BasicsC/string-buffer.h"
#include "ShapedJson/shaped-json.h"
#include "Utils/Exception.h"
#include "VocBase/replication-common.h"
#include "VocBase/voc-types.h"
#include "VocBase/vocbase.h"
// -----------------------------------------------------------------------------
// --SECTION-- forward declarations
// -----------------------------------------------------------------------------
struct TRI_shape_s;
struct TRI_string_buffer_s;
struct TRI_vocbase_col_s;
struct TRI_vocbase_s;
// -----------------------------------------------------------------------------
// --SECTION-- REPLICATION LOGGER
@ -58,18 +59,43 @@ struct TRI_vocbase_s;
/// @brief replication dump container
////////////////////////////////////////////////////////////////////////////////
typedef struct TRI_replication_dump_s {
struct TRI_string_buffer_s* _buffer;
struct TRI_replication_dump_t {
TRI_replication_dump_t (TRI_vocbase_t* vocbase,
size_t chunkSize)
: _vocbase(vocbase),
_buffer(nullptr),
_chunkSize(chunkSize),
_lastFoundTick(0),
_lastSid(0),
_lastShape(nullptr),
_collectionNames(),
_failed(false),
_bufferFull(false),
_hasMore(false) {
_buffer = TRI_CreateSizedStringBuffer(TRI_UNKNOWN_MEM_ZONE, chunkSize);
if (_buffer == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
}
~TRI_replication_dump_t () {
TRI_FreeStringBuffer(TRI_UNKNOWN_MEM_ZONE, _buffer);
_buffer = nullptr;
}
TRI_vocbase_t* _vocbase;
TRI_string_buffer_t* _buffer;
size_t _chunkSize;
TRI_voc_tick_t _lastFoundTick;
TRI_shape_sid_t _lastSid;
struct TRI_shape_s const* _lastShape;
struct TRI_vocbase_s* _vocbase;
TRI_associative_pointer_t _collectionNames;
std::unordered_map<TRI_voc_cid_t, std::string> _collectionNames;
bool _failed;
bool _hasMore;
bool _bufferFull;
}
TRI_replication_dump_t;
bool _hasMore;
};
// -----------------------------------------------------------------------------
// --SECTION-- public functions
@ -83,7 +109,6 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t*,
struct TRI_vocbase_col_s*,
TRI_voc_tick_t,
TRI_voc_tick_t,
uint64_t,
bool,
bool);
@ -94,22 +119,7 @@ int TRI_DumpCollectionReplication (TRI_replication_dump_t*,
int TRI_DumpLogReplication (struct TRI_vocbase_s*,
TRI_replication_dump_t*,
TRI_voc_tick_t,
TRI_voc_tick_t,
uint64_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief initialise a replication dump container
////////////////////////////////////////////////////////////////////////////////
int TRI_InitDumpReplication (TRI_replication_dump_t*,
struct TRI_vocbase_s*,
size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief destroy a replication dump container
////////////////////////////////////////////////////////////////////////////////
void TRI_DestroyDumpReplication (TRI_replication_dump_t*);
TRI_voc_tick_t);
#endif

4
arangod/VocBase/vocbase.cpp
View File

@ -1839,8 +1839,8 @@ char* TRI_GetCollectionNameByIdVocBase (TRI_vocbase_t* vocbase,
TRI_vocbase_col_t* found = static_cast<TRI_vocbase_col_t*>(TRI_LookupByKeyAssociativePointer(&vocbase->_collectionsById, &id));
if (found == NULL) {
name = NULL;
if (found == nullptr) {
name = nullptr;
}
else {
name = TRI_DuplicateStringZ(TRI_UNKNOWN_MEM_ZONE, found->_name);

6
arangod/Wal/Marker.h
View File

@ -275,11 +275,7 @@ namespace triagens {
inline char* end () const {
return _buffer + _size;
}
/*
inline char* payload () const {
return begin() + sizeof(TRI_df_marker_t);
}
*/
////////////////////////////////////////////////////////////////////////////////
/// @brief return the size of the marker
////////////////////////////////////////////////////////////////////////////////

6
js/apps/system/aardvark/frontend/js/client/bootstrap/module-internal.js
View File

@ -160,10 +160,12 @@
var i;
if (typeof body !== 'string') {
body = JSON.stringify(body);
internal.startCaptureMode();
print(body);
body = internal.stopCaptureMode();
}
curl = "unix> curl ";
curl = "shell> curl ";
if (method === 'POST') {
response = internal.arango.POST_RAW(url, body, headers);

8
js/apps/system/aardvark/frontend/js/modules/org/arangodb/general-graph.js
View File

@ -1548,8 +1548,8 @@ var _extendEdgeDefinitions = function (edgeDefinition) {
/// The creation of a graph requires the name of the graph and a definition of its edges.
///
/// For every type of edge definition a convenience method exists that can be used to create a graph.
/// Optionaly a list of vertex collections can be added, which are not used in any edge definition.
/// These collections are refered to as orphan collections within this chapter.
/// Optionally a list of vertex collections can be added, which are not used in any edge definition.
/// These collections are referred to as orphan collections within this chapter.
/// All collections used within the creation process are created if they do not exist.
///
/// *Parameter*
@ -2009,7 +2009,7 @@ var sortEdgeDefinition = function(edgeDefinition) {
/// g.relation.save("male/bob", "female/alice", {type: "married", _key: "bobAndAlice"});
/// @END_EXAMPLE_ARANGOSH_OUTPUT
///
/// If the collections of *from* and *to* are not defined in an edgeDefinition of the graph,
/// If the collections of *from* and *to* are not defined in an edge definition of the graph,
/// the edge will not be stored.
///
///
@ -2226,7 +2226,7 @@ var checkIfMayBeDropped = function(colName, graphName, graphs) {
/// *Remove a graph*
///
/// A graph can be dropped by its name.
/// This will automatically drop al collections contained in the graph as
/// This will automatically drop all collections contained in the graph as
/// long as they are not used within other graphs.
/// To drop the collections, the optional parameter *drop-collections* can be set to *true*.
///

6
js/client/bootstrap/module-internal.js
View File

@ -160,10 +160,12 @@
var i;
if (typeof body !== 'string') {
body = JSON.stringify(body);
internal.startCaptureMode();
print(body);
body = internal.stopCaptureMode();
}
curl = "unix> curl ";
curl = "shell> curl ";
if (method === 'POST') {
response = internal.arango.POST_RAW(url, body, headers);

2
js/common/modules/org/arangodb/general-graph.js
View File

@ -2008,7 +2008,7 @@ var sortEdgeDefinition = function(edgeDefinition) {
/// g.relation.save("male/bob", "female/alice", {type: "married", _key: "bobAndAlice"});
/// @END_EXAMPLE_ARANGOSH_OUTPUT
///
/// If the collections of *from* and *to* are not defined in an edgeDefinition of the graph,
/// If the collections of *from* and *to* are not defined in an edge definition of the graph,
/// the edge will not be stored.
///
///

3
js/server/modules/org/arangodb/foxx/base_middleware.js
View File

@ -61,7 +61,8 @@ BaseMiddleware = function () {
////////////////////////////////////////////////////////////////////////////////
body: function () {
return JSON.parse(this.requestBody);
var requestBody = this.requestBody || '{}';
return JSON.parse(requestBody);
},
////////////////////////////////////////////////////////////////////////////////

4
js/server/modules/org/arangodb/foxx/model.js
View File

@ -102,6 +102,7 @@ Model = function (attributes) {
this.attributes = whitelistProperties(attributes, this.constructor.attributes, true);
this.attributes = fillInDefaults(this.attributes, this.constructor.attributes);
this.whitelistedAttributes = whitelistProperties(this.attributes, this.constructor.attributes);
};
Model.fromClient = function (attributes) {
@ -269,8 +270,7 @@ _.extend(Model.prototype, {
forClient: function () {
'use strict';
var result = whitelistProperties(this.attributes, this.constructor.attributes);
return result;
return this.whitelistedAttributes;
}
});