Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

"doCompact" attribute for collections

This commit is contained in:
Jan Steemann 2013-06-19 16:51:16 +02:00
parent 2bd699045f
commit 9dfb7483ff
12 changed files with 170 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG
View File

@ -1,6 +1,9 @@
v1.4
----
* added "doCompact" attribute when creating collections and to collection.properties().
The attribute controls whether collection datafiles are compacted.
* changed the HTTP return code from 400 to 404 for some cases when there is a referral
to a non-existing collection or document.

22
UnitTests/HttpInterface/api-collection-spec.rb
View File

@ -258,6 +258,7 @@ describe ArangoDB do
doc.parsed_response['id'].should eq(@cid)
doc.parsed_response['name'].should eq(@cn)
doc.parsed_response['status'].should eq(3)
doc.parsed_response['doCompact'].should eq(true)
doc.parsed_response['waitForSync'].should eq(true)
doc.parsed_response['isVolatile'].should eq(false)
doc.parsed_response['isSystem'].should eq(false)
@ -944,6 +945,7 @@ describe ArangoDB do
doc.parsed_response['id'].should eq(cid)
doc.parsed_response['name'].should eq(cn)
doc.parsed_response['status'].should eq(3)
doc.parsed_response['doCompact'].should eq(true)
doc.parsed_response['waitForSync'].should eq(true)
doc.parsed_response['isVolatile'].should eq(false)
doc.parsed_response['isSystem'].should eq(false)
@ -961,6 +963,24 @@ describe ArangoDB do
doc.parsed_response['id'].should eq(cid)
doc.parsed_response['name'].should eq(cn)
doc.parsed_response['status'].should eq(3)
doc.parsed_response['doCompact'].should eq(true)
doc.parsed_response['waitForSync'].should eq(false)
doc.parsed_response['isVolatile'].should eq(false)
doc.parsed_response['isSystem'].should eq(false)
doc.parsed_response['keyOptions']['type'].should eq("traditional")
doc.parsed_response['keyOptions']['allowUserKeys'].should eq(true)
body = "{ \"doCompact\" : false }"
doc = ArangoDB.log_put("#{prefix}-identifier-properties-no-compact", cmd, :body => body)
doc.code.should eq(200)
doc.headers['content-type'].should eq("application/json; charset=utf-8")
doc.parsed_response['error'].should eq(false)
doc.parsed_response['code'].should eq(200)
doc.parsed_response['id'].should eq(cid)
doc.parsed_response['name'].should eq(cn)
doc.parsed_response['status'].should eq(3)
doc.parsed_response['doCompact'].should eq(false)
doc.parsed_response['waitForSync'].should eq(false)
doc.parsed_response['isVolatile'].should eq(false)
doc.parsed_response['isSystem'].should eq(false)
@ -1048,6 +1068,7 @@ describe ArangoDB do
doc.headers['content-type'].should eq("application/json; charset=utf-8")
doc.parsed_response['error'].should eq(false)
doc.parsed_response['code'].should eq(200)
doc.parsed_response['doCompact'].should eq(true)
doc.parsed_response['waitForSync'].should eq(false)
doc.parsed_response['isVolatile'].should eq(true)
doc.parsed_response['keyOptions']['type'].should eq("traditional")
@ -1078,6 +1099,7 @@ describe ArangoDB do
doc.parsed_response['id'].should eq(cid)
doc.parsed_response['name'].should eq(cn)
doc.parsed_response['status'].should eq(3)
doc.parsed_response['doCompact'].should eq(true)
doc.parsed_response['waitForSync'].should eq(true)
doc.parsed_response['isVolatile'].should eq(false)
doc.parsed_response['keyOptions']['type'].should eq("traditional")

20
arangod/V8Server/v8-vocbase.cpp
View File

@ -1495,6 +1495,14 @@ static v8::Handle<v8::Value> CreateVocBase (v8::Arguments const& argv, TRI_col_t
parameter._waitForSync = TRI_ObjectToBoolean(p->Get(v8g->WaitForSyncKey));
}
if (p->Has(v8g->DoCompactKey)) {
parameter._doCompact = TRI_ObjectToBoolean(p->Get(v8g->DoCompactKey));
}
else {
// default value for compaction
parameter._doCompact = true;
}
if (p->Has(v8g->IsSystemKey)) {
parameter._isSystem = TRI_ObjectToBoolean(p->Get(v8g->IsSystemKey));
}
@ -5023,11 +5031,17 @@ static v8::Handle<v8::Value> JS_PropertiesVocbaseCol (v8::Arguments const& argv)
// get the old values
TRI_LOCK_JOURNAL_ENTRIES_DOC_COLLECTION(document);
bool waitForSync = base->_info._waitForSync;
size_t maximalSize = base->_info._maximalSize;
bool doCompact = base->_info._doCompact;
bool waitForSync = base->_info._waitForSync;
TRI_UNLOCK_JOURNAL_ENTRIES_DOC_COLLECTION(document);
// extract doCompact flag
if (po->Has(v8g->DoCompactKey)) {
doCompact = TRI_ObjectToBoolean(po->Get(v8g->DoCompactKey));
}
// extract sync flag
if (po->Has(v8g->WaitForSyncKey)) {
waitForSync = TRI_ObjectToBoolean(po->Get(v8g->WaitForSyncKey));
@ -5059,6 +5073,7 @@ static v8::Handle<v8::Value> JS_PropertiesVocbaseCol (v8::Arguments const& argv)
// update collection
TRI_col_info_t newParameter;
newParameter._doCompact = doCompact;
newParameter._maximalSize = maximalSize;
newParameter._waitForSync = waitForSync;
@ -5078,8 +5093,9 @@ static v8::Handle<v8::Value> JS_PropertiesVocbaseCol (v8::Arguments const& argv)
if (TRI_IS_DOCUMENT_COLLECTION(base->_info._type)) {
TRI_json_t* keyOptions = primary->_keyGenerator->toJson(primary->_keyGenerator);
result->Set(v8g->IsVolatileKey, base->_info._isVolatile ? v8::True() : v8::False());
result->Set(v8g->DoCompactKey, base->_info._doCompact ? v8::True() : v8::False());
result->Set(v8g->IsSystemKey, base->_info._isSystem ? v8::True() : v8::False());
result->Set(v8g->IsVolatileKey, base->_info._isVolatile ? v8::True() : v8::False());
result->Set(v8g->JournalSizeKey, v8::Number::New(base->_info._maximalSize));
if (keyOptions != 0) {
result->Set(v8g->KeyOptionsKey, TRI_ObjectJson(keyOptions)->ToObject());

9
arangod/VocBase/collection.c
View File

@ -895,6 +895,7 @@ void TRI_InitCollectionInfo (TRI_vocbase_t* vocbase,
parameter->_tick = 0;
parameter->_deleted = false;
parameter->_doCompact = true;
parameter->_isVolatile = false;
parameter->_isSystem = false;
parameter->_maximalSize = (maximalSize / PageSize) * PageSize;
@ -921,6 +922,7 @@ void TRI_CopyCollectionInfo (TRI_col_info_t* dst, const TRI_col_info_t* const sr
dst->_tick = src->_tick;
dst->_deleted = src->_deleted;
dst->_doCompact = src->_doCompact;
dst->_isSystem = src->_isSystem;
dst->_isVolatile = src->_isVolatile;
dst->_maximalSize = src->_maximalSize;
@ -1146,6 +1148,8 @@ int TRI_LoadCollectionInfo (char const* path,
size_t n;
memset(parameter, 0, sizeof(TRI_col_info_t));
parameter->_doCompact = true;
parameter->_isVolatile = false;
// find parameter file
filename = TRI_Concatenate2File(path, TRI_COL_PARAMETER_FILE);
@ -1227,6 +1231,9 @@ int TRI_LoadCollectionInfo (char const* path,
if (TRI_EqualString(key->_value._string.data, "deleted")) {
parameter->_deleted = value->_value._boolean;
}
else if (TRI_EqualString(key->_value._string.data, "doCompact")) {
parameter->_doCompact = value->_value._boolean;
}
else if (TRI_EqualString(key->_value._string.data, "isVolatile")) {
parameter->_isVolatile = value->_value._boolean;
}
@ -1281,6 +1288,7 @@ int TRI_SaveCollectionInfo (char const* path,
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "type", TRI_CreateNumberJson(TRI_CORE_MEM_ZONE, info->_type));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "cid", TRI_CreateStringCopyJson(TRI_CORE_MEM_ZONE, cidString));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "deleted", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, info->_deleted));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "doCompact", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, info->_doCompact));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "maximalSize", TRI_CreateNumberJson(TRI_CORE_MEM_ZONE, info->_maximalSize));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "name", TRI_CreateStringCopyJson(TRI_CORE_MEM_ZONE, info->_name));
TRI_Insert3ArrayJson(TRI_CORE_MEM_ZONE, json, "isVolatile", TRI_CreateBooleanJson(TRI_CORE_MEM_ZONE, info->_isVolatile));
@ -1324,6 +1332,7 @@ int TRI_UpdateCollectionInfo (TRI_vocbase_t* vocbase,
}
if (parameter != 0) {
collection->_info._doCompact = parameter->_doCompact;
collection->_info._maximalSize = parameter->_maximalSize;
collection->_info._waitForSync = parameter->_waitForSync;

1
arangod/VocBase/collection.h
View File

@ -226,6 +226,7 @@ typedef struct TRI_col_info_s {
// flags
bool _deleted : 1; // if true, collection has been deleted
bool _doCompact: 1; // if true, collection will be compacted
bool _isSystem : 1; // if true, this is a system collection
bool _isVolatile : 1; // if true, collection is memory-only
bool _waitForSync : 1; // if true, wait for msync

10
arangod/VocBase/compactor.c
View File

@ -862,6 +862,7 @@ void TRI_CompactorVocBase (void* data) {
for (i = 0; i < n; ++i) {
TRI_vocbase_col_t* collection;
TRI_primary_collection_t* primary;
bool doCompact;
collection = collections._buffer[i];
@ -878,14 +879,15 @@ void TRI_CompactorVocBase (void* data) {
continue;
}
worked = false;
type = primary->base._info._type;
worked = false;
doCompact = primary->base._info._doCompact;
type = primary->base._info._type;
// for document collection, compactify datafiles
if (TRI_IS_DOCUMENT_COLLECTION(type)) {
if (collection->_status == TRI_VOC_COL_STATUS_LOADED) {
if (collection->_status == TRI_VOC_COL_STATUS_LOADED && doCompact) {
TRI_barrier_t* ce;
// check whether someone else holds a read-lock on the compaction lock
if (! TRI_TryWriteLockReadWriteLock(&primary->_compactionLock)) {
// someone else is holding the compactor lock, we'll not compact

20
arangod/VocBase/primary-collection.h
View File

@ -136,7 +136,7 @@ TRI_doc_mptr_t;
////////////////////////////////////////////////////////////////////////////////
typedef struct TRI_doc_datafile_info_s {
TRI_voc_fid_t _fid;
TRI_voc_fid_t _fid;
TRI_voc_ssize_t _numberAlive;
TRI_voc_ssize_t _numberDead;
@ -283,25 +283,25 @@ TRI_doc_collection_info_t;
////////////////////////////////////////////////////////////////////////////////
typedef struct TRI_primary_collection_s {
TRI_collection_t base;
TRI_collection_t base;
// .............................................................................
// this lock protects the _primaryIndex plus the _allIndexes
// and _headers attributes in derived types
// .............................................................................
TRI_read_write_lock_t _lock;
TRI_read_write_lock_t _lock;
TRI_shaper_t* _shaper;
TRI_barrier_list_t _barrierList;
TRI_associative_pointer_t _datafileInfo;
TRI_shaper_t* _shaper;
TRI_barrier_list_t _barrierList;
TRI_associative_pointer_t _datafileInfo;
TRI_associative_pointer_t _primaryIndex;
struct TRI_key_generator_s* _keyGenerator;
TRI_associative_pointer_t _primaryIndex;
struct TRI_key_generator_s* _keyGenerator;
struct TRI_cap_constraint_s* _capConstraint;
int64_t _numberDocuments;
TRI_read_write_lock_t _compactionLock;
int64_t _numberDocuments;
TRI_read_write_lock_t _compactionLock;
int (*beginRead) (struct TRI_primary_collection_s*);
int (*endRead) (struct TRI_primary_collection_s*);

23
js/actions/api-collection.js
View File

@ -55,6 +55,7 @@ function collectionRepresentation (collection, showProperties, showCount, showFi
if (showProperties) {
var properties = collection.properties();
result.doCompact = properties.doCompact;
result.isVolatile = properties.isVolatile;
result.isSystem = properties.isSystem;
result.journalSize = properties.journalSize;
@ -111,6 +112,9 @@ function collectionRepresentation (collection, showProperties, showCount, showFi
/// the data is synchronised to disk before returning from a create or
/// update of a document.
///
/// - `doCompact` (optional, default is `true`): whether or not the collection
/// will be compacted.
///
/// - `journalSize` (optional, default is a @ref
/// CommandLineArangod "configuration parameter"): The maximal size of
/// a journal or datafile. Note that this also limits the maximal
@ -202,14 +206,13 @@ function post_api_collection (req, res) {
var name = body.name;
var parameter = { waitForSync : false };
var type = arangodb.ArangoCollection.TYPE_DOCUMENT;
var cid;
if (body.hasOwnProperty("waitForSync")) {
parameter.waitForSync = body.waitForSync;
}
if (body.hasOwnProperty("journalSize")) {
parameter.journalSize = body.journalSize;
if (body.hasOwnProperty("doCompact")) {
parameter.doCompact = body.doCompact;
}
if (body.hasOwnProperty("isSystem")) {
@ -219,9 +222,9 @@ function post_api_collection (req, res) {
if (body.hasOwnProperty("isVolatile")) {
parameter.isVolatile = body.isVolatile;
}
if (body.hasOwnProperty("_id")) {
cid = body._id;
if (body.hasOwnProperty("journalSize")) {
parameter.journalSize = body.journalSize;
}
if (body.hasOwnProperty("type")) {
@ -379,13 +382,15 @@ function get_api_collections (req, res) {
///
/// @RESTDESCRIPTION
/// In addition to the above, the result will always contain the
/// `waitForSync`, `journalSize`, and `isVolatile` properties.
/// `waitForSync`, `doCompact`, `journalSize`, and `isVolatile` properties.
/// This is achieved by forcing a load of the underlying collection.
///
/// - `waitForSync`: If `true` then creating or changing a
/// document will wait until the data has been synchronised to disk.
///
/// - `journalSize`: The maximal size of a journal / datafile.
/// - `doCompact`: Whether or not the collection will be compacted.
///
/// - `journalSize`: The maximal size setting for journals / datafiles.
///
/// - `isVolatile`: If `true` then the collection data will be
/// kept in memory only and ArangoDB will not write or sync the data

7
js/common/tests/shell-collection.js
View File

@ -309,6 +309,7 @@ function CollectionSuite () {
assertEqual(false, p.waitForSync);
assertEqual(false, p.isVolatile);
assertEqual(true, p.doCompact);
db._drop(cn);
},
@ -333,6 +334,7 @@ function CollectionSuite () {
assertEqual(false, p.waitForSync);
assertEqual(false, p.isVolatile);
assertEqual(true, p.doCompact);
db._drop(cn);
},
@ -394,6 +396,7 @@ function CollectionSuite () {
assertEqual(false, p.waitForSync);
assertEqual(false, p.isVolatile);
assertEqual(1048576, p.journalSize);
assertEqual(true, p.doCompact);
db._drop(cn);
},
@ -419,6 +422,7 @@ function CollectionSuite () {
assertEqual(true, p.waitForSync);
assertEqual(false, p.isVolatile);
assertEqual(1048576, p.journalSize);
assertEqual(true, p.doCompact);
db._drop(cn);
},
@ -431,7 +435,7 @@ function CollectionSuite () {
var cn = "example";
db._drop(cn);
var c1 = db._create(cn, { isVolatile : true });
var c1 = db._create(cn, { isVolatile : true, doCompact: false });
assertTypeOf("string", c1._id);
assertEqual(cn, c1.name());
@ -443,6 +447,7 @@ function CollectionSuite () {
assertEqual(true, p.isVolatile);
assertEqual(1048576, p.journalSize);
assertEqual(false, p.doCompact);
db._drop(cn);
},

73
js/server/tests/compaction.js
View File

@ -124,6 +124,79 @@ function CompactionSuite () {
internal.db._drop(cn);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test figures after truncate and rotate, with compaction disabled
////////////////////////////////////////////////////////////////////////////////
testFiguresNoCompact : function () {
var maxWait;
var cn = "example";
var n = 400;
var payload = "the quick brown fox jumped over the lazy dog. a quick dog jumped over the lazy fox";
for (var i = 0; i < 5; ++i) {
payload += payload;
}
internal.db._drop(cn);
var c1 = internal.db._create(cn, { "journalSize" : 1048576, "doCompact" : false } );
for (var i = 0; i < n; ++i) {
c1.save({ _key: "test" + i, value : i, payload : payload });
}
c1.unload();
internal.wait(5);
var fig = c1.figures();
assertEqual(n, c1.count());
assertEqual(n, fig["alive"]["count"]);
assertEqual(0, fig["dead"]["count"]);
assertEqual(0, fig["dead"]["size"]);
assertEqual(0, fig["dead"]["deletion"]);
assertEqual(1, fig["journals"]["count"]);
assertTrue(0 < fig["datafiles"]["count"]);
c1.truncate();
c1.rotate();
var fig = c1.figures();
assertEqual(0, c1.count());
assertEqual(0, fig["alive"]["count"]);
assertTrue(0 < fig["dead"]["count"]);
assertTrue(0 < fig["dead"]["count"]);
assertTrue(0 < fig["dead"]["size"]);
assertTrue(0 < fig["dead"]["deletion"]);
assertEqual(1, fig["journals"]["count"]);
assertTrue(0 < fig["datafiles"]["count"]);
// wait for compactor to run
require("console").log("waiting for compactor to run");
// set max wait time
if (internal.valgrind) {
maxWait = 120;
}
else {
maxWait = 15;
}
internal.wait(maxWait);
fig = c1.figures();
assertEqual(0, c1.count());
assertEqual(0, fig["alive"]["count"]);
assertTrue(0 < fig["dead"]["count"]);
assertTrue(0 < fig["dead"]["count"]);
assertTrue(0 < fig["dead"]["size"]);
assertTrue(0 < fig["dead"]["deletion"]);
assertEqual(1, fig["journals"]["count"]);
assertTrue(0 < fig["datafiles"]["count"]);
internal.db._drop(cn);
},
////////////////////////////////////////////////////////////////////////////////
/// @brief test document presence after compaction
////////////////////////////////////////////////////////////////////////////////

2
lib/V8/v8-globals.cpp
View File

@ -63,6 +63,7 @@ TRI_v8_global_s::TRI_v8_global_s (v8::Isolate* isolate)
ClientKey(),
CodeKey(),
ContentTypeKey(),
DoCompactKey(),
DomainKey(),
ErrorKey(),
ErrorMessageKey(),
@ -127,6 +128,7 @@ TRI_v8_global_s::TRI_v8_global_s (v8::Isolate* isolate)
CodeKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("code"));
ContentTypeKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("contentType"));
CookiesKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("cookies"));
DoCompactKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("doCompact"));
DomainKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("domain"));
ErrorKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("error"));
ErrorMessageKey = v8::Persistent<v8::String>::New(isolate, TRI_V8_SYMBOL("errorMessage"));

6
lib/V8/v8-globals.h
View File

@ -355,6 +355,12 @@ typedef struct TRI_v8_global_s {
v8::Persistent<v8::String> CookiesKey;
////////////////////////////////////////////////////////////////////////////////
/// @brief "doCompact" key name
////////////////////////////////////////////////////////////////////////////////
v8::Persistent<v8::String> DoCompactKey;
////////////////////////////////////////////////////////////////////////////////
/// @brief "domain" key
////////////////////////////////////////////////////////////////////////////////