Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

renamed and removed

This commit is contained in:
Jan Steemann 2016-03-01 12:34:03 +01:00
parent e0e84b7ea1
commit 15350cf4b8
27 changed files with 223 additions and 775 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arangod/Aql/Query.cpp
View File

@ -30,7 +30,6 @@
#include "Aql/Parser.h"
#include "Aql/QueryCache.h"
#include "Aql/QueryList.h"
#include "Aql/ShortStringStorage.h"
#include "Basics/Exceptions.h"
#include "Basics/JsonHelper.h"
#include "Basics/WorkMonitor.h"

3
arangod/CMakeLists.txt
View File

@ -218,7 +218,6 @@ add_executable(${BIN_ARANGOD}
Scheduler/TaskManager.cpp
Scheduler/TimerTask.cpp
Statistics/statistics.cpp
Storage/Marker.cpp
Utils/CollectionExport.cpp
Utils/CollectionKeys.cpp
Utils/CollectionKeysRepository.cpp
@ -269,7 +268,7 @@ add_executable(${BIN_ARANGOD}
VocBase/datafile.cpp
VocBase/document-collection.cpp
VocBase/edge-collection.cpp
VocBase/headers.cpp
VocBase/MasterPointers.cpp
VocBase/replication-applier.cpp
VocBase/replication-common.cpp
VocBase/replication-dump.cpp

137
arangod/Storage/Marker.cpp
View File

@ -1,137 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "Storage/Marker.h"
#include "Basics/Exceptions.h"
using namespace arangodb;
// returns a type name for a marker
char const* MarkerHelper::typeName(MarkerType type) {
switch (type) {
case MarkerTypeHeader:
return "header";
case MarkerTypeFooter:
return "footer";
case MarkerTypeDocumentPreface:
return "document preface";
case MarkerTypeDocument:
return "document";
case MarkerTypeDocumentDeletion:
return "document deletion";
case MarkerTypeTransactionBegin:
return "transaction begin";
case MarkerTypeTransactionCommit:
return "transaction commit";
case MarkerTypeTransactionAbort:
return "transaction abort";
case MarkerTypeCollectionCreate:
return "collection create";
case MarkerTypeCollectionDrop:
return "collection drop";
case MarkerTypeCollectionRename:
return "collection rename";
case MarkerTypeCollectionProperties:
return "collection properties";
case MarkerTypeIndexCreate:
return "index create";
case MarkerTypeIndexDrop:
return "index drop";
case MarkerTypeDatabaseCreate:
return "database create";
case MarkerTypeDatabaseDrop:
return "database drop";
case MarkerMax:
break;
}
return "invalid marker type";
}
// returns the static length for the marker type
// the static length is the total length of the marker's static data fields,
// excluding the base marker's fields and excluding the marker's dynamic
// VPack data values
uint64_t MarkerHelper::staticLength(MarkerType type) {
switch (type) {
case MarkerTypeHeader:
case MarkerTypeFooter:
return MarkerReaderMeta::staticLength();
case MarkerTypeDocumentPreface:
return MarkerReaderDocumentPreface::staticLength();
case MarkerTypeDocument:
case MarkerTypeDocumentDeletion:
return MarkerReaderDocument::staticLength();
case MarkerTypeTransactionBegin:
case MarkerTypeTransactionCommit:
case MarkerTypeTransactionAbort:
return MarkerReaderTransaction::staticLength();
case MarkerTypeCollectionCreate:
case MarkerTypeCollectionDrop:
case MarkerTypeCollectionRename:
case MarkerTypeCollectionProperties:
return MarkerReaderCollection::staticLength();
case MarkerTypeIndexCreate:
case MarkerTypeIndexDrop:
return MarkerReaderIndex::staticLength();
case MarkerTypeDatabaseCreate:
case MarkerTypeDatabaseDrop:
return MarkerReaderDatabase::staticLength();
case MarkerMax:
break;
}
THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
}
// calculate the required length for a marker of the specified type, given a
// payload of the specified length
uint64_t MarkerHelper::calculateMarkerLength(MarkerType type,
uint64_t payloadLength) {
uint64_t bodyLength = staticLength(type) + payloadLength;
return calculateHeaderLength(bodyLength) + bodyLength;
}
// calculate the required length for the header of a marker, given a body
// of the specified length
uint64_t MarkerHelper::calculateHeaderLength(uint64_t bodyLength) {
if (bodyLength < (1 << (3 * 8))) {
return 16;
}
return 24;
}
std::ostream& operator<<(std::ostream& stream,
arangodb::MarkerReader const* marker) {
stream << "[Marker " << MarkerHelper::typeName(marker->type())
<< ", size: " << marker->length() << "]";
return stream;
}

502
arangod/Storage/Marker.h
View File

@ -1,502 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_STORAGE_MARKER_H
#define ARANGOD_STORAGE_MARKER_H 1
#include "Basics/Common.h"
#include "Basics/hashes.h"
#include <velocypack/Options.h>
#include <velocypack/Slice.h>
#include <velocypack/velocypack-aliases.h>
namespace arangodb {
////////////////////////////////////////////////////////////////////////////////
/// @brief available marker types. values must be < 128
////////////////////////////////////////////////////////////////////////////////
enum MarkerType : uint8_t {
MarkerTypeHeader = 1,
MarkerTypeFooter = 2,
MarkerTypeDocumentPreface = 10,
MarkerTypeDocument = 11,
MarkerTypeDocumentDeletion = 12,
MarkerTypeTransactionBegin = 20,
MarkerTypeTransactionCommit = 21,
MarkerTypeTransactionAbort = 22,
MarkerTypeCollectionCreate = 30,
MarkerTypeCollectionDrop = 31,
MarkerTypeCollectionRename = 32,
MarkerTypeCollectionProperties = 33,
MarkerTypeIndexCreate = 40,
MarkerTypeIndexDrop = 41,
MarkerTypeDatabaseCreate = 50,
MarkerTypeDatabaseDrop = 51,
MarkerMax = 127
};
static_assert(MarkerMax < 128, "invalid maximum marker type value");
struct MarkerHelper {
uint32_t alignedSize(uint32_t value) { return ((value + 7) / 8) * 8; }
uint64_t alignedSize(uint64_t value) { return ((value + 7) / 8) * 8; }
template <typename T>
static inline uint32_t calculateNumberLength(T value) throw() {
uint32_t len = 1;
while (value > 0) {
++len;
value >>= 8;
++len;
}
return len;
}
template <typename T>
static inline T readNumber(uint8_t const* source, uint32_t length) {
T value = 0;
uint64_t x = 0;
uint8_t const* end = source + length;
do {
value += static_cast<T>(*source++) << x;
x += 8;
} while (source < end);
return value;
}
template <typename T>
static inline void storeNumber(uint8_t* dest, T value, uint32_t length) {
uint8_t* end = dest + length;
do {
*dest++ = static_cast<uint8_t>(value & 0xff);
value >>= 8;
} while (dest < end);
}
// returns a type name for a marker
static char const* typeName(MarkerType type);
// returns the static length for the marker type
// the static length is the total length of the marker's static data fields,
// excluding the base marker's fields and excluding the marker's dynamic
// VPack data values
static uint64_t staticLength(MarkerType type);
// calculate the required length for a marker of the specified type, given a
// payload of the specified length
static uint64_t calculateMarkerLength(MarkerType type,
uint64_t payloadLength);
// calculate the required length for the header of a marker, given a body
// of the specified length
static uint64_t calculateHeaderLength(uint64_t bodyLength);
};
/* the base layout for all markers is:
uint32_t type and length information (first byte contains marker type,
following 3 bytes contain length information)
uint32_t crc checksum
uint64_t tick value
(uint64_t) optional length information
char[] payload
if the highest bit in the first byte (type) is set, then the length of
the marker is coded in the uint64_t length value at offset 0x10.
if the highest bit in the first byte (type) is not set, then the length
of the marker is coded in bytes from offset 1 to (including) 3.
*/
class MarkerReader {
public:
static uint64_t const MinMarkerLength = 16;
MarkerReader(uint8_t* begin) : _begin(begin), _length(calculateLength()) {
TRI_ASSERT(_length >= MinMarkerLength);
}
public:
char* data() const { return reinterpret_cast<char*>(_begin); }
uint8_t* begin() const { return _begin; }
uint8_t* end() const { return _begin + _length; }
MarkerType type() const {
// read lowest 7 bits of head byte
uint8_t type = *_begin & 0x7f;
return static_cast<MarkerType>(type);
}
uint32_t length() const { return static_cast<uint32_t>(_length); }
uint32_t headerLength() const {
if (*_begin & 0x80) {
return 24;
}
return 16;
}
// gets the currently persisted CRC value of the marker
uint32_t persistedCrc() const {
return readAlignedNumber<uint32_t>(_begin + 4, 4);
}
// recalculates the actual CRC value of the marker
uint32_t actualCrc() const {
static uint32_t const empty = 0;
uint32_t crc = TRI_InitialCrc32();
crc = TRI_BlockCrc32(crc, data(), 4); // calculate crc for first 4 bytes
crc = TRI_BlockCrc32(crc, reinterpret_cast<char const*>(&empty),
sizeof(empty));
crc = TRI_BlockCrc32(crc, data() + 8,
_length - 8); // calculate crc for rest of marker
crc = TRI_FinalCrc32(crc);
return crc;
}
uint64_t tick() const { return readAlignedNumber<uint64_t>(_begin + 8, 8); }
uint8_t* payload() const { return _begin + headerLength(); }
template <typename T>
T readNumber(uint8_t const* start, uint64_t length) const {
return MarkerHelper::readNumber<T>(start, static_cast<uint32_t>(length));
}
template <typename T>
T readAlignedNumber(uint8_t const* start, uint64_t length) const {
TRI_ASSERT(reinterpret_cast<uintptr_t>(start) % sizeof(T) == 0);
// TODO: create an optimized version for aligned data access
return readNumber<T>(start, length);
}
protected:
uint64_t calculateLength() const {
if (*_begin & 0x80) {
return readAlignedNumber<uint64_t>(_begin + 8, 8);
}
return readNumber<uint64_t>(_begin + 1, 3);
}
protected:
uint8_t* _begin;
uint64_t _length;
};
class MarkerWriter : public MarkerReader {
public:
MarkerWriter(uint8_t* begin) : MarkerReader(begin) {}
public:
// calculates the marker's CRC values and stores it
uint32_t storeCrc() {
// invalidate crc data in marker
MarkerHelper::storeNumber(_begin + 4, 0, 4);
// recalculate crc
uint32_t crc = TRI_InitialCrc32();
crc = TRI_BlockCrc32(crc, data(), _length);
crc = TRI_FinalCrc32(crc);
MarkerHelper::storeNumber(_begin + 4, crc, 4);
return crc;
}
template <typename T>
void storeNumber(uint8_t* start, T value, uint64_t length) const {
MarkerHelper::storeNumber<T>(start, value, static_cast<uint32_t>(length));
}
template <typename T>
void storeAlignedNumber(uint8_t* start, T value, uint64_t length) const {
TRI_ASSERT(reinterpret_cast<uintptr_t>(start) % sizeof(T) == 0);
// TODO: create an optimized version for aligned data access
storeNumber<T>(start, value, length);
}
};
template <typename T>
class MarkerAccessorMeta : public T {
/* this is a marker for meta data (header, footer etc)
its layout is:
BaseMarker base (16 or 24 bytes)
*/
public:
MarkerAccessorMeta(uint8_t* begin) : T(begin) {}
static uint64_t staticLength() { return 0; }
};
typedef MarkerAccessorMeta<MarkerReader> MarkerReaderMeta;
class MarkerWriterMeta : public MarkerAccessorMeta<MarkerWriter> {
public:
MarkerWriterMeta(uint8_t* begin) : MarkerAccessorMeta<MarkerWriter>(begin) {}
};
template <typename T>
class MarkerAccessorDocumentPreface : public T {
/* this is a preface marker for documents operations
its layout is:
BaseMarker base (16 or 24 bytes)
uint64_t database id
uint64_t collection id
*/
public:
MarkerAccessorDocumentPreface(uint8_t* begin) : T(begin) {}
public:
uint64_t database() const {
return MarkerReader::readAlignedNumber<uint64_t>(MarkerReader::payload(),
8);
}
uint64_t collection() const {
return MarkerReader::readAlignedNumber<uint64_t>(
MarkerReader::payload() + 8, 8);
}
static uint64_t staticLength() { return 16; }
};
typedef MarkerAccessorDocumentPreface<MarkerReader> MarkerReaderDocumentPreface;
class MarkerWriterDocumentPreface
: public MarkerAccessorDocumentPreface<MarkerWriter> {
public:
MarkerWriterDocumentPreface(uint8_t* begin)
: MarkerAccessorDocumentPreface<MarkerWriter>(begin) {}
public:
void database(uint64_t id) {
MarkerWriter::storeAlignedNumber<uint64_t>(MarkerWriter::payload(), id, 8);
}
void collection(uint64_t id) {
MarkerWriter::storeAlignedNumber<uint64_t>(MarkerWriter::payload() + 8, id,
8);
}
};
template <typename T>
class MarkerAccessorDocument : public T {
/* this is a combined marker for documents / edges and deletions.
its layout is:
BaseMarker base (16 or 24 bytes)
uint64_t transaction id
VersionedVPack VPack with document value
VersionedVPack is one byte for the VPack version, followed
by the actual VPack value
*/
public:
MarkerAccessorDocument(uint8_t* begin) : T(begin) {}
public:
uint64_t transaction() const {
return MarkerReader::readAlignedNumber<uint64_t>(MarkerReader::payload(),
8);
}
uint8_t* versionedVPackValue() const { return this->payload() + 8; }
uint8_t* vPackValue() const { return versionedVPackValue() + 1; }
VPackSlice slice() const { return VPackSlice(this->vpackValue(), &VPackOptions::Defaults); }
static uint64_t staticLength() { return 8; }
};
typedef MarkerAccessorDocument<MarkerReader> MarkerReaderDocument;
class MarkerWriterDocument : public MarkerAccessorDocument<MarkerWriter> {
public:
MarkerWriterDocument(uint8_t* begin)
: MarkerAccessorDocument<MarkerWriter>(begin) {}
public:
void transaction(uint64_t tid) const {
MarkerWriter::storeAlignedNumber<uint64_t>(MarkerReader::payload(), tid, 8);
}
};
template <typename T>
class MarkerAccessorTransaction : public T {
/* this is a marker accessor for transaction handling.
its layout is:
BaseMarker base (16 or 24 bytes)
uint64_t transaction id
*/
public:
MarkerAccessorTransaction(uint8_t* begin) : T(begin) {}
public:
uint64_t transaction() const {
return MarkerReader::readAlignedNumber<uint64_t>(MarkerReader::payload(),
8);
}
static uint64_t staticLength() { return 0; }
};
typedef MarkerAccessorTransaction<MarkerReader> MarkerReaderTransaction;
class MarkerWriterTransaction : public MarkerAccessorTransaction<MarkerWriter> {
/* this is a marker accessor for transaction handling.
its layout is:
BaseMarker base (16 or 24 bytes)
uint64_t transaction id
*/
public:
MarkerWriterTransaction(uint8_t* begin)
: MarkerAccessorTransaction<MarkerWriter>(begin) {}
public:
void transaction(uint64_t tid) const {
MarkerWriter::storeAlignedNumber<uint64_t>(MarkerReader::payload(), tid, 8);
}
};
template <typename T>
class MarkerAccessorStructural : public T {
/* this is a marker accessor for structural data
(i.e. collections, indexes, databases)
its layout is:
BaseMarker base (16 or 24 bytes)
VersionedVPack VPack with document value
VersionedVPack is one byte for the VPack version, followed
by the actual VPack value
*/
public:
MarkerAccessorStructural(uint8_t* begin) : T(begin) {}
public:
uint8_t* versionedVPackValue() const { return MarkerReader::payload() + 8; }
uint8_t* vPackValue() const { return versionedVPackValue() + 1; }
VPackSlice slice() const { return VPackSlice(this->vpackValue(), &VPackOptions::Defaults); }
static uint64_t staticLength() { return 0; }
};
template <typename T>
class MarkerAccessorDatabase : public MarkerAccessorStructural<T> {
/* this is a marker accessor for databases.
its layout is:
BaseMarker base (16 or 24 bytes)
VersionedVPack VPack with document value
VersionedVPack is one byte for the VPack version, followed
by the actual VPack value
*/
public:
MarkerAccessorDatabase(uint8_t* begin) : MarkerAccessorStructural<T>(begin) {}
static uint64_t staticLength() { return 0; }
};
typedef MarkerAccessorDatabase<MarkerReader> MarkerReaderDatabase;
class MarkerWriterDatabase : public MarkerAccessorDatabase<MarkerWriter> {
public:
MarkerWriterDatabase(uint8_t* begin)
: MarkerAccessorDatabase<MarkerWriter>(begin) {}
};
template <typename T>
class MarkerAccessorCollection : public MarkerAccessorStructural<T> {
/* this is a marker accessor for collections.
its layout is:
BaseMarker base (16 or 24 bytes)
VersionedVPack VPack with more data value
VersionedVPack is one byte for the VPack version, followed
by the actual VPack value
*/
public:
MarkerAccessorCollection(uint8_t* begin)
: MarkerAccessorStructural<T>(begin) {}
static uint64_t staticLength() { return 0; }
};
typedef MarkerAccessorCollection<MarkerReader> MarkerReaderCollection;
class MarkerWriterCollection : public MarkerAccessorCollection<MarkerWriter> {
public:
MarkerWriterCollection(uint8_t* begin)
: MarkerAccessorCollection<MarkerWriter>(begin) {}
};
template <typename T>
class MarkerAccessorIndex : public MarkerAccessorStructural<T> {
/* this is a marker accessor for indexes.
its layout is:
BaseMarker base (16 or 24 bytes)
uint64_t transaction id
*/
public:
MarkerAccessorIndex(uint8_t* begin) : MarkerAccessorStructural<T>(begin) {}
static uint64_t staticLength() { return 0; }
};
typedef MarkerAccessorIndex<MarkerReader> MarkerReaderIndex;
class MarkerWriterIndex : public MarkerReaderIndex {
public:
MarkerWriterIndex(uint8_t* begin) : MarkerReaderIndex(begin) {}
};
}
std::ostream& operator<<(std::ostream&, arangodb::MarkerReader const*);
#endif

11
arangod/Utils/Transaction.cpp
View File

@ -24,11 +24,12 @@
#include "Utils/transactions.h"
#include "Basics/conversions.h"
#include "Basics/StringUtils.h"
#include "Indexes/PrimaryIndex.h"
#include "Storage/Marker.h"
#include "Utils/OperationCursor.h"
#include "VocBase/KeyGenerator.h"
#include "Cluster/ClusterMethods.h"
#include "Indexes/PrimaryIndex.h"
#include "Utils/OperationCursor.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/KeyGenerator.h"
#include "VocBase/MasterPointers.h"
#include <velocypack/Builder.h>
#include <velocypack/Collection.h>
@ -785,7 +786,7 @@ OperationResult Transaction::insertLocal(std::string const& collectionName,
// add _id attribute
uint8_t* p = merge.add(TRI_VOC_ATTRIBUTE_ID, VPackValuePair(9ULL, VPackValueType::Custom));
*p++ = 0xf3; // custom type for _id
MarkerHelper::storeNumber<uint64_t>(p, cid, sizeof(uint64_t));
DatafileHelper::StoreNumber<uint64_t>(p, cid, sizeof(uint64_t));
merge.close();

1
arangod/Utils/Transaction.h
View File

@ -39,7 +39,6 @@
#include "VocBase/Ditch.h"
#include "VocBase/document-collection.h"
#include "VocBase/edge-collection.h"
#include "VocBase/headers.h"
#include "VocBase/transaction.h"
#include "VocBase/update-policy.h"
#include "VocBase/vocbase.h"

6
arangod/Utils/TransactionContext.cpp
View File

@ -22,7 +22,7 @@
////////////////////////////////////////////////////////////////////////////////
#include "TransactionContext.h"
#include "Storage/Marker.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/Ditch.h"
#include "VocBase/document-collection.h"
#include "Wal/LogfileManager.h"
@ -53,7 +53,7 @@ struct CustomTypeHandler : public VPackCustomTypeHandler {
"invalid value type");
}
uint64_t cid = MarkerHelper::readNumber<uint64_t>(value.begin() + 1, sizeof(uint64_t));
uint64_t cid = DatafileHelper::ReadNumber<uint64_t>(value.begin() + 1, sizeof(uint64_t));
char buffer[512]; // This is enough for collection name + _key
size_t len = resolver->getCollectionName(&buffer[0], cid);
buffer[len] = '/';
@ -82,7 +82,7 @@ struct CustomTypeHandler : public VPackCustomTypeHandler {
"invalid value type");
}
uint64_t cid = MarkerHelper::readNumber<uint64_t>(value.begin() + 1, sizeof(uint64_t));
uint64_t cid = DatafileHelper::ReadNumber<uint64_t>(value.begin() + 1, sizeof(uint64_t));
std::string result(resolver->getCollectionName(cid));
result.push_back('/');
VPackSlice key = base.get(TRI_VOC_ATTRIBUTE_KEY);

6
arangod/V8Server/V8VPackWrapper.cpp
View File

@ -24,13 +24,13 @@
#include "V8VPackWrapper.h"
#include "Basics/conversions.h"
#include "Basics/Logger.h"
#include "Storage/Marker.h"
#include "Utils/Transaction.h"
#include "V8/v8-conv.h"
#include "V8/v8-globals.h"
#include "V8/v8-vpack.h"
#include "V8Server/v8-vocbaseprivate.h"
#include "VocBase/datafile.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/document-collection.h"
#include "VocBase/KeyGenerator.h"
@ -63,7 +63,7 @@ static int const SLOT_DITCH = 2;
////////////////////////////////////////////////////////////////////////////////
static inline VPackSlice VPackFromMarker(TRI_df_marker_t const* marker) {
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + VPackOffset(marker->_type);
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + DatafileHelper::VPackOffset(marker->_type);
return VPackSlice(ptr);
}
@ -79,7 +79,7 @@ static void AddCollectionId(v8::Isolate* isolate, v8::Handle<v8::Object> self,
// extract cid from marker
VPackSlice id = slice.get(TRI_VOC_ATTRIBUTE_ID);
uint64_t cid = MarkerHelper::readNumber<uint64_t>(id.begin() + 1, sizeof(uint64_t));
uint64_t cid = DatafileHelper::ReadNumber<uint64_t>(id.begin() + 1, sizeof(uint64_t));
VPackValueLength keyLength;
char const* key = slice.get(TRI_VOC_ATTRIBUTE_KEY).getString(keyLength);

1
arangod/V8Server/v8-collection.cpp
View File

@ -29,7 +29,6 @@
#include "Basics/ScopeGuard.h"
#include "Cluster/ClusterMethods.h"
#include "Indexes/PrimaryIndex.h"
#include "Storage/Marker.h"
#include "Utils/OperationOptions.h"
#include "Utils/OperationResult.h"
#include "Utils/transactions.h"

99
arangod/VocBase/DatafileHelper.h Normal file
View File

@ -0,0 +1,99 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_VOC_BASE_DATAFILE_HELPER_H
#define ARANGOD_VOC_BASE_DATAFILE_HELPER_H 1
#include "Basics/Common.h"
#include "VocBase/datafile.h"
namespace arangodb {
namespace DatafileHelper {
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the 8-byte aligned size for the value
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline T AlignedSize(T value) {
return (value + 7) - ((value + 7) & 7);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the 8-byte aligned size for the marker
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline T AlignedMarkerSize(TRI_df_marker_t const* marker) {
size_t value = marker->_size;
return static_cast<T>((value + 7) - ((value + 7) & 7));
}
////////////////////////////////////////////////////////////////////////////////
/// @brief portably and safely reads a number
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline T ReadNumber(uint8_t const* source, uint32_t length) {
T value = 0;
uint64_t x = 0;
uint8_t const* end = source + length;
do {
value += static_cast<T>(*source++) << x;
x += 8;
} while (source < end);
return value;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief portably and safely stores a number
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline void StoreNumber(uint8_t* dest, T value, uint32_t length) {
uint8_t* end = dest + length;
do {
*dest++ = static_cast<uint8_t>(value & 0xff);
value >>= 8;
} while (dest < end);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the marker-specific offset to the vpack payload
////////////////////////////////////////////////////////////////////////////////
static inline size_t VPackOffset(TRI_df_marker_type_t type) {
auto t = static_cast<TRI_df_marker_type_e>(type);
if (t == TRI_WAL_MARKER_VPACK_DOCUMENT ||
t == TRI_WAL_MARKER_VPACK_REMOVE) {
return sizeof(TRI_df_marker_t) + sizeof(TRI_voc_tid_t);
}
TRI_ASSERT(false);
return 0;
}
}
} // namespace
#endif

1
arangod/VocBase/KeyGenerator.cpp
View File

@ -32,6 +32,7 @@
#include <array>
using namespace arangodb;
using namespace arangodb::basics;
////////////////////////////////////////////////////////////////////////////////

4
arangod/VocBase/KeyGenerator.h
View File

@ -41,7 +41,7 @@ namespace velocypack {
class Builder;
class Slice;
}
}
class KeyGenerator {
public:
//////////////////////////////////////////////////////////////////////////////
@ -253,6 +253,8 @@ class AutoIncrementKeyGenerator : public KeyGenerator {
uint64_t _increment; // increment value
};
}
////////////////////////////////////////////////////////////////////////////////
/// @brief validate a document id (collection name + / + document key)
////////////////////////////////////////////////////////////////////////////////

37
arangod/VocBase/headers.cpp → arangod/VocBase/MasterPointers.cpp
View File

@ -21,10 +21,13 @@
/// @author Dr. Frank Celler
////////////////////////////////////////////////////////////////////////////////
#include "headers.h"
#include "MasterPointers.h"
#include "Basics/Logger.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/document-collection.h"
using namespace arangodb;
////////////////////////////////////////////////////////////////////////////////
/// @brief get the size (number of entries) for a block, based on a function
///
@ -53,7 +56,7 @@ static inline size_t GetBlockSize(size_t blockNumber) {
/// @brief creates the headers
////////////////////////////////////////////////////////////////////////////////
TRI_headers_t::TRI_headers_t()
MasterPointers::MasterPointers()
: _freelist(nullptr),
_nrAllocated(0),
_nrLinked(0),
@ -66,7 +69,7 @@ TRI_headers_t::TRI_headers_t()
/// @brief destroys the headers
////////////////////////////////////////////////////////////////////////////////
TRI_headers_t::~TRI_headers_t() {
MasterPointers::~MasterPointers() {
for (auto& it : _blocks) {
delete[] it;
}
@ -76,7 +79,7 @@ TRI_headers_t::~TRI_headers_t() {
/// @brief returns the memory usage
////////////////////////////////////////////////////////////////////////////////
uint64_t TRI_headers_t::memory() const {
uint64_t MasterPointers::memory() const {
return _nrAllocated * sizeof(TRI_doc_mptr_t);
}
@ -85,7 +88,7 @@ uint64_t TRI_headers_t::memory() const {
/// this is called when there is an update operation on a document
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::moveBack(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
void MasterPointers::moveBack(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
if (header == nullptr) {
return;
}
@ -103,7 +106,7 @@ void TRI_headers_t::moveBack(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old)
int64_t oldSize = static_cast<int64_t>(old->getMarkerPtr()->_size);
// we must adjust the size of the collection
_totalSize += (AlignedSize<int64_t>(newSize) - AlignedSize<int64_t>(oldSize));
_totalSize += (DatafileHelper::AlignedSize<int64_t>(newSize) - DatafileHelper::AlignedSize<int64_t>(oldSize));
TRI_ASSERT(_totalSize > 0);
}
@ -113,7 +116,7 @@ void TRI_headers_t::moveBack(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old)
/// (specified in "old"), note that this is only used in revert operations
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::move(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
void MasterPointers::move(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
if (header == nullptr) {
return;
}
@ -131,14 +134,14 @@ void TRI_headers_t::move(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
// update operation. The "new" document is removed again and the "old"
// one is used once more. Therefore, the signs in the following statement
// are actually OK:
_totalSize -= (AlignedSize<int64_t>(newSize) - AlignedSize<int64_t>(oldSize));
_totalSize -= (DatafileHelper::AlignedSize<int64_t>(newSize) - DatafileHelper::AlignedSize<int64_t>(oldSize));
}
////////////////////////////////////////////////////////////////////////////////
/// @brief unlinks a header from the linked list, without freeing it
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::unlink(TRI_doc_mptr_t* header) {
void MasterPointers::unlink(TRI_doc_mptr_t* header) {
TRI_ASSERT(header != nullptr);
TRI_ASSERT(header->getDataPtr() != nullptr);
@ -147,7 +150,7 @@ void TRI_headers_t::unlink(TRI_doc_mptr_t* header) {
TRI_ASSERT(_nrLinked > 0);
_nrLinked--;
_totalSize -= AlignedSize<int64_t>(size);
_totalSize -= DatafileHelper::AlignedSize<int64_t>(size);
if (_nrLinked == 0) {
TRI_ASSERT(_totalSize == 0);
@ -161,7 +164,7 @@ void TRI_headers_t::unlink(TRI_doc_mptr_t* header) {
/// (specified in "old")
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::relink(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
void MasterPointers::relink(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
if (header == nullptr) {
return;
}
@ -172,7 +175,7 @@ void TRI_headers_t::relink(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
this->move(header, old);
_nrLinked++;
_totalSize += AlignedSize<int64_t>(size);
_totalSize += DatafileHelper::AlignedSize<int64_t>(size);
TRI_ASSERT(_totalSize > 0);
}
@ -180,7 +183,7 @@ void TRI_headers_t::relink(TRI_doc_mptr_t* header, TRI_doc_mptr_t const* old) {
/// @brief requests a new header
////////////////////////////////////////////////////////////////////////////////
TRI_doc_mptr_t* TRI_headers_t::request(size_t size) {
TRI_doc_mptr_t* MasterPointers::request(size_t size) {
TRI_doc_mptr_t* header;
TRI_ASSERT(size > 0);
@ -229,7 +232,7 @@ TRI_doc_mptr_t* TRI_headers_t::request(size_t size) {
_nrAllocated++;
_nrLinked++;
_totalSize += AlignedSize<int64_t>(size);
_totalSize += DatafileHelper::AlignedSize<int64_t>(size);
return result;
}
@ -238,7 +241,7 @@ TRI_doc_mptr_t* TRI_headers_t::request(size_t size) {
/// @brief releases a header, putting it back onto the freelist
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::release(TRI_doc_mptr_t* header, bool unlinkHeader) {
void MasterPointers::release(TRI_doc_mptr_t* header, bool unlinkHeader) {
if (header == nullptr) {
return;
}
@ -275,9 +278,9 @@ void TRI_headers_t::release(TRI_doc_mptr_t* header, bool unlinkHeader) {
/// this is called by the collector
////////////////////////////////////////////////////////////////////////////////
void TRI_headers_t::adjustTotalSize(int64_t oldSize, int64_t newSize) {
void MasterPointers::adjustTotalSize(int64_t oldSize, int64_t newSize) {
// oldSize = size of marker in WAL
// newSize = size of marker in datafile
_totalSize -= (AlignedSize<int64_t>(oldSize) - AlignedSize<int64_t>(newSize));
_totalSize -= (DatafileHelper::AlignedSize<int64_t>(oldSize) - DatafileHelper::AlignedSize<int64_t>(newSize));
}

18
arangod/VocBase/headers.h → arangod/VocBase/MasterPointers.h
View File

@ -21,29 +21,31 @@
/// @author Dr. Frank Celler
////////////////////////////////////////////////////////////////////////////////
#ifndef ARANGOD_VOC_BASE_HEADERS_H
#define ARANGOD_VOC_BASE_HEADERS_H 1
#ifndef ARANGOD_VOC_BASE_MASTER_POINTERS_H
#define ARANGOD_VOC_BASE_MASTER_POINTERS_H 1
#include "Basics/Common.h"
struct TRI_doc_mptr_t;
class TRI_headers_t {
TRI_headers_t(TRI_headers_t const&) = delete;
TRI_headers_t& operator=(TRI_headers_t const&) = delete;
namespace arangodb {
class MasterPointers {
MasterPointers(MasterPointers const&) = delete;
MasterPointers& operator=(MasterPointers const&) = delete;
public:
//////////////////////////////////////////////////////////////////////////////
/// @brief creates the headers
//////////////////////////////////////////////////////////////////////////////
TRI_headers_t();
MasterPointers();
//////////////////////////////////////////////////////////////////////////////
/// @brief destroys the headers
//////////////////////////////////////////////////////////////////////////////
~TRI_headers_t();
~MasterPointers();
public:
//////////////////////////////////////////////////////////////////////////////
@ -119,4 +121,6 @@ class TRI_headers_t {
std::vector<TRI_doc_mptr_t const*> _blocks;
};
}
#endif

3
arangod/VocBase/collection.cpp
View File

@ -36,6 +36,7 @@
#include "Basics/random.h"
#include "Basics/tri-strings.h"
#include "Cluster/ClusterInfo.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
#include "VocBase/vocbase.h"
@ -409,7 +410,7 @@ static bool CheckCollection(TRI_collection_t* collection, bool ignoreErrors) {
char* ptr = datafile->_data;
// skip the datafile header
ptr += AlignedSize<size_t>(sizeof(TRI_df_header_marker_t));
ptr += DatafileHelper::AlignedSize<size_t>(sizeof(TRI_df_header_marker_t));
cm = (TRI_col_header_marker_t*)ptr;
if (cm->base._type != TRI_COL_MARKER_HEADER) {

13
arangod/VocBase/compactor.cpp
View File

@ -36,6 +36,7 @@
#include "Indexes/PrimaryIndex.h"
#include "Utils/StandaloneTransactionContext.h"
#include "Utils/transactions.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/DatafileStatistics.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
@ -389,7 +390,7 @@ static bool Compactifier(TRI_df_marker_t const* marker, void* data,
// new or updated document
if (marker->_type == TRI_WAL_MARKER_VPACK_DOCUMENT) {
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
TRI_ASSERT(slice.isObject());
VPackSlice const keySlice(slice.get(TRI_VOC_ATTRIBUTE_KEY));
TRI_voc_rid_t const rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
@ -404,7 +405,7 @@ static bool Compactifier(TRI_df_marker_t const* marker, void* data,
if (deleted) {
// found a dead document
context->_dfi.numberDead++;
context->_dfi.sizeDead += AlignedMarkerSize<int64_t>(marker);
context->_dfi.sizeDead += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
return true;
}
@ -430,7 +431,7 @@ static bool Compactifier(TRI_df_marker_t const* marker, void* data,
}
context->_dfi.numberAlive++;
context->_dfi.sizeAlive += AlignedMarkerSize<int64_t>(marker);
context->_dfi.sizeAlive += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
// deletions
@ -536,7 +537,7 @@ static bool CalculateSize(TRI_df_marker_t const* marker, void* data,
// new or updated document
if (marker->_type == TRI_WAL_MARKER_VPACK_DOCUMENT) {
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
TRI_ASSERT(slice.isObject());
VPackSlice const keySlice(slice.get(TRI_VOC_ATTRIBUTE_KEY));
TRI_voc_rid_t const rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
@ -552,12 +553,12 @@ static bool CalculateSize(TRI_df_marker_t const* marker, void* data,
}
context->_keepDeletions = true;
context->_targetSize += AlignedMarkerSize<int64_t>(marker);
context->_targetSize += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
// deletions
else if (marker->_type == TRI_WAL_MARKER_VPACK_REMOVE) {
context->_targetSize += AlignedMarkerSize<int64_t>(marker);
context->_targetSize += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
return true;

26
arangod/VocBase/datafile.cpp
View File

@ -31,12 +31,14 @@
#include "Basics/hashes.h"
#include "Basics/memory-map.h"
#include "Basics/tri-strings.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/server.h"
#include <sstream>
// #define DEBUG_DATAFILE 1
using namespace arangodb;
using namespace arangodb::basics;
////////////////////////////////////////////////////////////////////////////////
@ -142,8 +144,7 @@ static std::string DiagnoseMarker(TRI_df_marker_t const* marker,
std::ostringstream result;
if (marker == nullptr) {
result << "marker is undefined. should not happen";
return result.str();
return "marker is undefined. should not happen";
}
// check marker type
@ -178,14 +179,13 @@ static std::string DiagnoseMarker(TRI_df_marker_t const* marker,
}
if (reinterpret_cast<char const*>(marker) + marker->_size > end) {
result << "marker size is beyond end of datafile";
return result.str();
return "marker size is beyond end of datafile";
}
TRI_voc_crc_t crc = CalculateCrcValue(marker);
if (marker->_crc == crc) {
result << "crc checksum is correct";
return "crc checksum is correct";
} else {
result << "crc checksum (hex " << std::hex << marker->_crc
<< ") is wrong. expecting (hex " << std::hex << crc << ")";
@ -534,7 +534,7 @@ static TRI_df_scan_t ScanDatafile(TRI_datafile_t const* datafile) {
entry._position = static_cast<TRI_voc_size_t>(ptr - datafile->_data);
entry._size = marker->_size;
entry._realSize = AlignedMarkerSize<size_t>(marker);
entry._realSize = DatafileHelper::AlignedMarkerSize<size_t>(marker);
entry._tick = marker->_tick;
entry._type = marker->_type;
entry._status = 1;
@ -607,12 +607,12 @@ static TRI_df_scan_t ScanDatafile(TRI_datafile_t const* datafile) {
entry._key = nullptr;
if (marker->_type == TRI_WAL_MARKER_VPACK_DOCUMENT) {
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
TRI_ASSERT(slice.isObject());
std::string const key(slice.get(TRI_VOC_ATTRIBUTE_KEY).copyString());
entry._key = TRI_DuplicateString(TRI_UNKNOWN_MEM_ZONE, key.c_str(), key.size());
} else if (marker->_type == TRI_WAL_MARKER_VPACK_REMOVE) {
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
TRI_ASSERT(slice.isObject());
std::string const key(slice.get(TRI_VOC_ATTRIBUTE_KEY).copyString());
entry._key = TRI_DuplicateString(TRI_UNKNOWN_MEM_ZONE, key.c_str(), key.size());
@ -620,7 +620,7 @@ static TRI_df_scan_t ScanDatafile(TRI_datafile_t const* datafile) {
TRI_PushBackVector(&scan._entries, &entry);
size_t size = AlignedMarkerSize<size_t>(marker);
size_t size = DatafileHelper::AlignedMarkerSize<size_t>(marker);
currentSize += static_cast<TRI_voc_size_t>(size);
if (marker->_type == TRI_DF_MARKER_FOOTER) {
@ -748,7 +748,7 @@ static bool TryRepairDatafile(TRI_datafile_t* datafile) {
}
}
size_t size = AlignedMarkerSize<TRI_voc_size_t>(marker);
size_t size = DatafileHelper::AlignedMarkerSize<TRI_voc_size_t>(marker);
currentSize += size;
if (marker->_type == TRI_DF_MARKER_FOOTER) {
@ -965,7 +965,7 @@ static bool CheckDatafile(TRI_datafile_t* datafile, bool ignoreFailures) {
maxTick = marker->_tick;
}
size_t size = AlignedMarkerSize<size_t>(marker);
size_t size = DatafileHelper::AlignedMarkerSize<size_t>(marker);
currentSize += size;
if (marker->_type == TRI_DF_MARKER_FOOTER) {
@ -1501,7 +1501,7 @@ int TRI_ReserveElementDatafile(TRI_datafile_t* datafile, TRI_voc_size_t size,
TRI_df_marker_t** position,
TRI_voc_size_t maximalJournalSize) {
*position = nullptr;
size = AlignedSize<TRI_voc_size_t>(size);
size = DatafileHelper::AlignedSize<TRI_voc_size_t>(size);
if (datafile->_state != TRI_DF_STATE_WRITE) {
if (datafile->_state == TRI_DF_STATE_READ) {
@ -1700,7 +1700,7 @@ bool TRI_IterateDatafile(TRI_datafile_t* datafile,
return false;
}
ptr += AlignedMarkerSize<size_t>(marker);
ptr += DatafileHelper::AlignedMarkerSize<size_t>(marker);
}
return true;

34
arangod/VocBase/datafile.h
View File

@ -517,40 +517,6 @@ char const* TRI_NameMarkerDatafile(TRI_df_marker_t const*);
void TRI_InitMarkerDatafile(char*, TRI_df_marker_type_e, TRI_voc_size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the 8-byte aligned size for the value
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline T AlignedSize(T value) {
return (value + 7) - ((value + 7) & 7);
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the 8-byte aligned size for the marker
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static inline T AlignedMarkerSize(TRI_df_marker_t const* marker) {
size_t value = marker->_size;
return static_cast<T>((value + 7) - ((value + 7) & 7));
}
////////////////////////////////////////////////////////////////////////////////
/// @brief returns the marker-specific offset to the vpack payload
////////////////////////////////////////////////////////////////////////////////
static inline size_t VPackOffset(TRI_df_marker_type_t type) {
auto t = static_cast<TRI_df_marker_type_e>(type);
if (t == TRI_WAL_MARKER_VPACK_DOCUMENT ||
t == TRI_WAL_MARKER_VPACK_REMOVE) {
return sizeof(TRI_df_marker_t) + sizeof(TRI_voc_tid_t);
}
TRI_ASSERT(false);
return 0;
}
////////////////////////////////////////////////////////////////////////////////
/// @brief checks whether a marker is valid
////////////////////////////////////////////////////////////////////////////////

23
arangod/VocBase/document-collection.cpp
View File

@ -45,11 +45,12 @@
#include "Utils/transactions.h"
#include "Utils/CollectionReadLocker.h"
#include "Utils/CollectionWriteLocker.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/Ditch.h"
#include "VocBase/edge-collection.h"
#include "VocBase/ExampleMatcher.h"
#include "VocBase/headers.h"
#include "VocBase/KeyGenerator.h"
#include "VocBase/MasterPointers.h"
#include "VocBase/server.h"
#include "VocBase/shape-accessor.h"
#include "VocBase/update-policy.h"
@ -817,7 +818,7 @@ static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker,
TRI_document_collection_t* document = state->_document;
arangodb::Transaction* trx = state->_trx;
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice const keySlice = slice.get(TRI_VOC_ATTRIBUTE_KEY);
std::string const key(keySlice.copyString());
TRI_voc_rid_t const rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
@ -867,7 +868,7 @@ static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker,
// update the datafile info
state->_dfi->numberAlive++;
state->_dfi->sizeAlive += AlignedMarkerSize<int64_t>(marker);
state->_dfi->sizeAlive += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
// it is an update, but only if found has a smaller revision identifier
@ -895,13 +896,13 @@ static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker,
int64_t size = static_cast<int64_t>(oldData.getMarkerPtr()->_size);
dfi->numberAlive--;
dfi->sizeAlive -= AlignedSize<int64_t>(size);
dfi->sizeAlive -= DatafileHelper::AlignedSize<int64_t>(size);
dfi->numberDead++;
dfi->sizeDead += AlignedSize<int64_t>(size);
dfi->sizeDead += DatafileHelper::AlignedSize<int64_t>(size);
}
state->_dfi->numberAlive++;
state->_dfi->sizeAlive += AlignedMarkerSize<int64_t>(marker);
state->_dfi->sizeAlive += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
// it is a stale update
@ -909,7 +910,7 @@ static int OpenIteratorHandleDocumentMarker(TRI_df_marker_t const* marker,
TRI_ASSERT(found->getDataPtr() != nullptr);
state->_dfi->numberDead++;
state->_dfi->sizeDead += AlignedSize<int64_t>(found->getMarkerPtr()->_size);
state->_dfi->sizeDead += DatafileHelper::AlignedSize<int64_t>(found->getMarkerPtr()->_size);
}
return TRI_ERROR_NO_ERROR;
@ -925,7 +926,7 @@ static int OpenIteratorHandleDeletionMarker(TRI_df_marker_t const* marker,
TRI_document_collection_t* document = state->_document;
arangodb::Transaction* trx = state->_trx;
VPackSlice const slice(reinterpret_cast<char const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
VPackSlice const slice(reinterpret_cast<char const*>(marker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
VPackSlice const keySlice = slice.get(TRI_VOC_ATTRIBUTE_KEY);
std::string const key(keySlice.copyString());
TRI_voc_rid_t const rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
@ -965,12 +966,12 @@ static int OpenIteratorHandleDeletionMarker(TRI_df_marker_t const* marker,
TRI_ASSERT(found->getDataPtr() != nullptr);
int64_t size = AlignedSize<int64_t>(found->getMarkerPtr()->_size);
int64_t size = DatafileHelper::AlignedSize<int64_t>(found->getMarkerPtr()->_size);
dfi->numberAlive--;
dfi->sizeAlive -= AlignedSize<int64_t>(size);
dfi->sizeAlive -= DatafileHelper::AlignedSize<int64_t>(size);
dfi->numberDead++;
dfi->sizeDead += AlignedSize<int64_t>(size);
dfi->sizeDead += DatafileHelper::AlignedSize<int64_t>(size);
state->_dfi->numberDeletions++;
DeletePrimaryIndex(trx, document, found, false);

11
arangod/VocBase/document-collection.h
View File

@ -30,9 +30,10 @@
#include "Cluster/ClusterInfo.h"
#include "Utils/OperationOptions.h"
#include "VocBase/collection.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/DatafileStatistics.h"
#include "VocBase/Ditch.h"
#include "VocBase/headers.h"
#include "VocBase/MasterPointers.h"
#include "VocBase/transaction.h"
#include "VocBase/update-policy.h"
#include "VocBase/voc-types.h"
@ -44,6 +45,7 @@ namespace arangodb {
class EdgeIndex;
class ExampleMatcher;
class Index;
class KeyGenerator;
class PrimaryIndex;
class Transaction;
namespace velocypack {
@ -52,7 +54,6 @@ class Slice;
}
}
class KeyGenerator;
////////////////////////////////////////////////////////////////////////////////
/// @brief master pointer
@ -115,7 +116,7 @@ struct TRI_doc_mptr_t {
TRI_df_marker_t const* marker =
static_cast<TRI_df_marker_t const*>(_dataptr);
return reinterpret_cast<uint8_t const*>(marker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT);
return reinterpret_cast<uint8_t const*>(marker) + arangodb::DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT);
}
//////////////////////////////////////////////////////////////////////////////
@ -261,8 +262,8 @@ struct TRI_document_collection_t : public TRI_collection_t {
mutable arangodb::Ditches _ditches;
TRI_headers_t _masterPointers;
KeyGenerator* _keyGenerator;
arangodb::MasterPointers _masterPointers;
arangodb::KeyGenerator* _keyGenerator;
std::vector<arangodb::Index*> _indexes;

15
arangod/VocBase/replication-dump.cpp
View File

@ -32,6 +32,7 @@
#include "Cluster/ServerState.h"
#include "Utils/CollectionNameResolver.h"
#include "VocBase/collection.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/datafile.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
@ -298,7 +299,7 @@ static int StringifyMarkerDump(TRI_replication_dump_t* dump,
case TRI_WAL_MARKER_VPACK_DOCUMENT: {
TRI_ASSERT(nullptr == document);
auto m = static_cast<wal::vpack_document_marker_t const*>(marker);
VPackSlice slice(reinterpret_cast<char const*>(m) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice slice(reinterpret_cast<char const*>(m) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
key = slice.get(TRI_VOC_ATTRIBUTE_KEY).getString(keyLength);
rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
type = REPLICATION_MARKER_DOCUMENT;
@ -311,7 +312,7 @@ static int StringifyMarkerDump(TRI_replication_dump_t* dump,
case TRI_WAL_MARKER_VPACK_REMOVE: {
TRI_ASSERT(nullptr == document);
auto m = static_cast<wal::vpack_remove_marker_t const*>(marker);
VPackSlice slice(reinterpret_cast<char const*>(m) + VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
VPackSlice slice(reinterpret_cast<char const*>(m) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
key = slice.get(TRI_VOC_ATTRIBUTE_KEY).getString(keyLength);
rid = std::stoull(slice.get(TRI_VOC_ATTRIBUTE_REV).copyString());
type = REPLICATION_MARKER_REMOVE;
@ -438,7 +439,7 @@ static int StringifyWalMarkerDocument(TRI_replication_dump_t* dump,
}
#endif
VPackSlice slice(reinterpret_cast<char const*>(VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT)));
VPackSlice slice(reinterpret_cast<char const*>(DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT)));
APPEND_STRING(dump->_buffer, "\"tid\":\"");
APPEND_UINT64(dump->_buffer, m->_transactionId);
@ -479,7 +480,7 @@ static int StringifyWalMarkerRemove(TRI_replication_dump_t* dump,
APPEND_UINT64(dump->_buffer, m->_transactionId);
APPEND_STRING(dump->_buffer, "\",\"key\":\"");
VPackSlice slice(reinterpret_cast<char const*>(VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT)));
VPackSlice slice(reinterpret_cast<char const*>(DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT)));
std::string key(slice.get(TRI_VOC_ATTRIBUTE_KEY).copyString());
TRI_AppendString2StringBuffer(dump->_buffer, key.c_str(), key.size());
@ -835,7 +836,7 @@ static int DumpCollection(TRI_replication_dump_t* dump,
break;
}
ptr += AlignedMarkerSize<size_t>(marker);
ptr += DatafileHelper::AlignedMarkerSize<size_t>(marker);
if (marker->_type == TRI_DF_MARKER_BLANK) {
// fully ignore these marker types. they don't need to be replicated,
@ -1020,7 +1021,7 @@ int TRI_DumpLogReplication(
break;
}
ptr += AlignedMarkerSize<size_t>(marker);
ptr += DatafileHelper::AlignedMarkerSize<size_t>(marker);
// get the marker's tick and check whether we should include it
TRI_voc_tick_t foundTick = marker->_tick;
@ -1149,7 +1150,7 @@ int TRI_DetermineOpenTransactionsReplication(TRI_replication_dump_t* dump,
break;
}
ptr += AlignedMarkerSize<size_t>(marker);
ptr += DatafileHelper::AlignedMarkerSize<size_t>(marker);
// get the marker's tick and check whether we should include it
TRI_voc_tick_t foundTick = marker->_tick;

9
arangod/VocBase/transaction.cpp
View File

@ -28,6 +28,7 @@
#include "Basics/Logger.h"
#include "Basics/tri-strings.h"
#include "Basics/Exceptions.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/collection.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
@ -47,6 +48,8 @@
#endif
using namespace arangodb;
////////////////////////////////////////////////////////////////////////////////
/// @brief returns whether the collection is currently locked
////////////////////////////////////////////////////////////////////////////////
@ -208,10 +211,10 @@ static void FreeOperations(TRI_transaction_t* trx) {
auto it2 = stats.find(fid);
if (it2 == stats.end()) {
stats.emplace(fid, std::make_pair(1, AlignedMarkerSize<int64_t>(marker)));
stats.emplace(fid, std::make_pair(1, DatafileHelper::AlignedMarkerSize<int64_t>(marker)));
} else {
(*it2).second.first++;
(*it2).second.second += AlignedMarkerSize<int64_t>(marker);
(*it2).second.second += DatafileHelper::AlignedMarkerSize<int64_t>(marker);
}
}
}
@ -1140,7 +1143,7 @@ int TRI_AddOperationTransaction(TRI_transaction_t* trx,
TRI_df_marker_t const* marker = static_cast<TRI_df_marker_t const*>(
operation.oldHeader.getDataPtr()); // PROTECTED by trx from above
document->_datafileStatistics.increaseDead(
operation.oldHeader._fid, 1, AlignedMarkerSize<int64_t>(marker));
operation.oldHeader._fid, 1, DatafileHelper::AlignedMarkerSize<int64_t>(marker));
}
} else {
// operation is buffered and might be rolled back

21
arangod/Wal/CollectorThread.cpp
View File

@ -35,6 +35,7 @@
#include "Utils/DatabaseGuard.h"
#include "Utils/StandaloneTransactionContext.h"
#include "Utils/transactions.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/DatafileStatistics.h"
#include "VocBase/document-collection.h"
#include "VocBase/server.h"
@ -189,7 +190,7 @@ static bool ScanMarker(TRI_df_marker_t const* marker, void* data,
break;
}
VPackSlice slice(reinterpret_cast<char const*>(m) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice slice(reinterpret_cast<char const*>(m) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
state->documentOperations[collectionId][slice.get(TRI_VOC_ATTRIBUTE_KEY).copyString()] = marker;
state->operationsCount[collectionId]++;
break;
@ -218,7 +219,7 @@ static bool ScanMarker(TRI_df_marker_t const* marker, void* data,
break;
}
VPackSlice slice(reinterpret_cast<char const*>(m) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice slice(reinterpret_cast<char const*>(m) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
state->documentOperations[collectionId][slice.get(TRI_VOC_ATTRIBUTE_KEY).copyString()] = marker;
state->operationsCount[collectionId]++;
break;
@ -687,7 +688,7 @@ void CollectorThread::processCollectionMarker(
auto& dfi = createDfi(cache, fid);
dfi.numberUncollected--;
VPackSlice slice(reinterpret_cast<char const*>(walMarker) + VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
VPackSlice slice(reinterpret_cast<char const*>(walMarker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_DOCUMENT));
TRI_ASSERT(slice.isObject());
TRI_voc_rid_t revisionId = arangodb::basics::VelocyPackHelper::stringUInt64(slice.get(TRI_VOC_ATTRIBUTE_REV));
@ -699,12 +700,12 @@ void CollectorThread::processCollectionMarker(
// somebody inserted a new revision of the document or the revision
// was already moved by the compactor
dfi.numberDead++;
dfi.sizeDead += AlignedSize<int64_t>(datafileMarkerSize);
dfi.sizeDead += DatafileHelper::AlignedSize<int64_t>(datafileMarkerSize);
} else {
// update size info
document->_masterPointers.adjustTotalSize(
AlignedSize<int64_t>(walMarker->_size),
AlignedSize<int64_t>(datafileMarkerSize));
DatafileHelper::AlignedSize<int64_t>(walMarker->_size),
DatafileHelper::AlignedSize<int64_t>(datafileMarkerSize));
// we can safely update the master pointer's dataptr value
found->setDataPtr(
@ -712,14 +713,14 @@ void CollectorThread::processCollectionMarker(
found->_fid = fid;
dfi.numberAlive++;
dfi.sizeAlive += AlignedSize<int64_t>(datafileMarkerSize);
dfi.sizeAlive += DatafileHelper::AlignedSize<int64_t>(datafileMarkerSize);
}
} else if (walMarker->_type == TRI_WAL_MARKER_VPACK_REMOVE) {
auto& dfi = createDfi(cache, fid);
dfi.numberUncollected--;
dfi.numberDeletions++;
VPackSlice slice(reinterpret_cast<char const*>(walMarker) + VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
VPackSlice slice(reinterpret_cast<char const*>(walMarker) + DatafileHelper::VPackOffset(TRI_WAL_MARKER_VPACK_REMOVE));
TRI_ASSERT(slice.isObject());
TRI_voc_rid_t revisionId = arangodb::basics::VelocyPackHelper::stringUInt64(slice.get(TRI_VOC_ATTRIBUTE_REV));
@ -729,7 +730,7 @@ void CollectorThread::processCollectionMarker(
if (found != nullptr && found->_rid > revisionId) {
// somebody re-created the document with a newer revision
dfi.numberDead++;
dfi.sizeDead += AlignedSize<int64_t>(datafileMarkerSize);
dfi.sizeDead += DatafileHelper::AlignedSize<int64_t>(datafileMarkerSize);
}
}
}
@ -1219,7 +1220,7 @@ char* CollectorThread::nextFreeMarkerPosition(
TRI_document_collection_t* document, TRI_voc_tick_t tick,
TRI_df_marker_type_e type, TRI_voc_size_t size, CollectorCache* cache) {
TRI_collection_t* collection = document;
size = AlignedSize<TRI_voc_size_t>(size);
size = DatafileHelper::AlignedSize<TRI_voc_size_t>(size);
char* dst = nullptr;
TRI_datafile_t* datafile = nullptr;

2
arangod/Wal/DocumentOperation.h
View File

@ -4,7 +4,7 @@
#include "Basics/Common.h"
#include "VocBase/document-collection.h"
#include "VocBase/headers.h"
#include "VocBase/MasterPointers.h"
#include "VocBase/voc-types.h"
#include "Wal/Marker.h"

4
arangod/Wal/Logfile.cpp
View File

@ -24,7 +24,9 @@
#include "Logfile.h"
#include "Basics/FileUtils.h"
#include "Basics/files.h"
#include "VocBase/DatafileHelper.h"
using namespace arangodb;
using namespace arangodb::wal;
////////////////////////////////////////////////////////////////////////////////
@ -152,7 +154,7 @@ int Logfile::judge(std::string const& filename) {
////////////////////////////////////////////////////////////////////////////////
char* Logfile::reserve(size_t size) {
size = AlignedSize<size_t>(size);
size = DatafileHelper::AlignedSize<size_t>(size);
char* result = _df->_next;

6
arangod/Wal/RecoverState.cpp
View File

@ -30,6 +30,7 @@
#include "Utils/SingleCollectionTransaction.h"
#include "Utils/StandaloneTransactionContext.h"
#include "VocBase/collection.h"
#include "VocBase/DatafileHelper.h"
#include "Wal/LogfileManager.h"
#include "Wal/Slots.h"
@ -37,6 +38,7 @@
#include <velocypack/Parser.h>
#include <velocypack/velocypack-aliases.h>
using namespace arangodb;
using namespace arangodb::wal;
////////////////////////////////////////////////////////////////////////////////
@ -545,7 +547,7 @@ bool RecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* data,
TRI_df_marker_t const* marker = static_cast<TRI_df_marker_t const*>(envelope->mem());
std::string const collectionName = trx->documentCollection()->_info.name();
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + VPackOffset(marker->_type);
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + DatafileHelper::VPackOffset(marker->_type);
OperationOptions options;
options.silent = true;
@ -602,7 +604,7 @@ bool RecoverState::ReplayMarker(TRI_df_marker_t const* marker, void* data,
TRI_df_marker_t const* marker = static_cast<TRI_df_marker_t const*>(envelope->mem());
std::string const collectionName = trx->documentCollection()->_info.name();
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + VPackOffset(marker->_type);
uint8_t const* ptr = reinterpret_cast<uint8_t const*>(marker) + DatafileHelper::VPackOffset(marker->_type);
OperationOptions options;
options.silent = true;

4
arangod/Wal/Slots.cpp
View File

@ -25,10 +25,12 @@
#include "Basics/ConditionLocker.h"
#include "Basics/MutexLocker.h"
#include "Basics/Logger.h"
#include "VocBase/DatafileHelper.h"
#include "VocBase/datafile.h"
#include "VocBase/server.h"
#include "Wal/LogfileManager.h"
using namespace arangodb;
using namespace arangodb::wal;
////////////////////////////////////////////////////////////////////////////////
@ -131,7 +133,7 @@ SlotInfo Slots::nextUnused(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId
static size_t const PrologueSize = sizeof(TRI_df_prologue_marker_t);
// we need to use the aligned size for writing
uint32_t alignedSize = AlignedSize<uint32_t>(size);
uint32_t alignedSize = DatafileHelper::AlignedSize<uint32_t>(size);
int iterations = 0;
bool hasWaited = false;
bool mustWritePrologue = false;