Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bug fix/fixes 1610 (#3425) 

* remove memory zone remainders

* remove useless calls

* attempt to fix non-deterministic CRC test

* attempt to fix non-deterministic replication tests

* fix document removal with multiple documents in cluster

* attempt to fix non-determinism of tests

* fix privilege checks for accessing collections by collection id, not by name
This commit is contained in:
Jan 2017-10-18 12:23:39 +02:00 committed by Frank Celler
parent a7903495a5
commit 2fee652e77
20 changed files with 111 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
UnitTests/HttpInterface/api-replication-rocksdb-spec.rb
View File

@ -511,7 +511,6 @@ describe ArangoDB do
after do
ArangoDB.delete(api + "/batch/#{@batchId}", :body => "")
ArangoDB.put(api + "/logger-stop", :body => "")
ArangoDB.drop_collection("UnitTestsReplication")
ArangoDB.drop_collection("UnitTestsReplication2")
end
@ -1632,7 +1631,6 @@ describe ArangoDB do
after do
ArangoDB.delete(api + "/batch/#{@batchId}", :body => "")
ArangoDB.put(api + "/logger-stop", :body => "")
ArangoDB.drop_collection("UnitTestsReplication", "UnitTestDB")
ArangoDB.drop_collection("UnitTestsReplication2", "UnitTestDB")
end

2
arangod/MMFiles/MMFilesDatafile.cpp
View File

@ -605,7 +605,7 @@ int MMFilesDatafile::writeElement(void* position, MMFilesMarker const* marker, b
TRI_IF_FAILURE("BreakHeaderMarker") {
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
if (marker->getType() == TRI_DF_MARKER_HEADER) {
if (marker->getType() == TRI_DF_MARKER_HEADER && getName().find("logfile-") == std::string::npos) {
// intentionally corrupt the marker
reinterpret_cast<MMFilesMarker*>(position)->breakIt();
}

6
arangod/MMFiles/MMFilesLogfileManager.cpp
View File

@ -831,7 +831,11 @@ int MMFilesLogfileManager::waitForCollectorQueue(TRI_voc_cid_t cid, double timeo
// this is useful to ensure that any open writes up to this point have made
// it into a logfile
int MMFilesLogfileManager::flush(bool waitForSync, bool waitForCollector,
bool writeShutdownFile) {
bool writeShutdownFile) {
TRI_IF_FAILURE("LogfileManagerFlush") {
return TRI_ERROR_NO_ERROR;
}
TRI_ASSERT(!_inRecovery);
MMFilesWalLogfile::IdType lastOpenLogfileId;

30
arangod/RestServer/DatabaseFeature.cpp
View File

@ -1018,6 +1018,36 @@ TRI_vocbase_t* DatabaseFeature::lookupDatabase(std::string const& name) {
return nullptr;
}
std::string DatabaseFeature::translateCollectionName(std::string const& dbName, std::string const& collectionName) {
if (ServerState::instance()->isCoordinator()) {
auto unuser(_databasesProtector.use());
auto theLists = _databasesLists.load();
auto it = theLists->_coordinatorDatabases.find(dbName);
if (it == theLists->_coordinatorDatabases.end()) {
return std::string();
}
TRI_vocbase_t* vocbase = (*it).second;
TRI_ASSERT(vocbase != nullptr);
TRI_ASSERT(vocbase->type() == TRI_VOCBASE_TYPE_COORDINATOR);
return vocbase->collectionName(StringUtils::uint64(collectionName));
} else {
auto unuser(_databasesProtector.use());
auto theLists = _databasesLists.load();
auto it = theLists->_databases.find(dbName);
if (it == theLists->_databases.end()) {
return std::string();
}
TRI_vocbase_t* vocbase = (*it).second;
TRI_ASSERT(vocbase != nullptr);
TRI_ASSERT(vocbase->type() == TRI_VOCBASE_TYPE_NORMAL);
return vocbase->collectionName(StringUtils::uint64(collectionName));
}
}
void DatabaseFeature::enumerateDatabases(std::function<void(TRI_vocbase_t*)> func) {
if (ServerState::instance()->isCoordinator()) {
auto unuser(_databasesProtector.use());

1
arangod/RestServer/DatabaseFeature.h
View File

@ -105,6 +105,7 @@ class DatabaseFeature final : public application_features::ApplicationFeature {
TRI_vocbase_t* lookupDatabaseCoordinator(std::string const& name);
TRI_vocbase_t* lookupDatabase(std::string const& name);
void enumerateDatabases(std::function<void(TRI_vocbase_t*)>);
std::string translateCollectionName(std::string const& dbName, std::string const& collectionName);
void useSystemDatabase();
TRI_vocbase_t* systemDatabase() const { return _vocbase; }

2
arangod/RocksDBEngine/RocksDBReplicationManager.cpp
View File

@ -246,7 +246,7 @@ void RocksDBReplicationManager::drop(TRI_vocbase_t* vocbase) {
}
}
garbageCollect(true);
garbageCollect(false);
}
////////////////////////////////////////////////////////////////////////////////

2
arangod/RocksDBEngine/RocksDBRestReplicationHandler.cpp
View File

@ -493,7 +493,7 @@ void RocksDBRestReplicationHandler::handleCommandBatch() {
}
// extract ttl
double expires = VelocyPackHelper::getNumericValue<double>(input->slice(), "ttl", 0);
double expires = VelocyPackHelper::getNumericValue<double>(input->slice(), "ttl", RocksDBReplicationContext::DefaultTTL);
int res = TRI_ERROR_NO_ERROR;
bool busy;

2
arangod/StorageEngine/TransactionState.cpp
View File

@ -113,7 +113,7 @@ int TransactionState::addCollection(TRI_voc_cid_t cid,
// avoid extra lookups of auth context, if we use the same db as stored
// in the execution context initialized by RestServer/VocbaseContext
AuthLevel level = auth->canUseCollection(ExecContext::CURRENT->user(),
_vocbase->name(), colName);
_vocbase->name(), colName);
if (level == AuthLevel::NONE) {
LOG_TOPIC(TRACE, Logger::AUTHORIZATION) << "User " << ExecContext::CURRENT->user()

1
arangod/Transaction/Methods.cpp
View File

@ -2099,7 +2099,6 @@ OperationResult transaction::Methods::removeLocal(
payload.add(StaticStrings::KeyString, s);
s = result.get(StaticStrings::RevString);
payload.add(StaticStrings::RevString, s);
TRI_SanitizeObject(doc, payload);
};
VPackSlice ourResult = resultBuilder.slice();

2
arangod/Utils/ExecContext.h
View File

@ -23,7 +23,7 @@
#ifndef ARANGOD_UTILS_EXECCONTEXT_H
#define ARANGOD_UTILS_EXECCONTEXT_H 1
#include <string>
#include "Basics/Common.h"
#include "Utils/Authentication.h"
namespace arangodb {

2
arangod/V8Server/v8-collection.cpp
View File

@ -3000,7 +3000,7 @@ static void JS_CollectionVocbase(
name);
if (level == AuthLevel::NONE) {
TRI_V8_THROW_EXCEPTION_MESSAGE(TRI_ERROR_FORBIDDEN,
"No access to collection");
std::string("No access to collection '") + name + "'");
}
}

25
arangod/VocBase/AuthInfo.cpp
View File

@ -891,6 +891,31 @@ AuthLevel AuthInfo::canUseDatabase(std::string const& username,
AuthLevel AuthInfo::canUseCollection(std::string const& username,
std::string const& dbname,
std::string const& coll) {
if (coll.empty()) {
// no collection name given
return AuthLevel::NONE;
}
if (coll[0] >= '0' && coll[0] <= '9') {
// lookup by collection id
// translate numeric collection id into collection name
return canUseCollectionInternal(username, dbname, DatabaseFeature::DATABASE->translateCollectionName(dbname, coll));
}
// lookup by collection name
return canUseCollectionInternal(username, dbname, coll);
}
// internal method called by canUseCollection
// asserts that collection name is non-empty and already translated
// from collection id to name
AuthLevel AuthInfo::canUseCollectionInternal(std::string const& username,
std::string const& dbname,
std::string const& coll) {
// we must have got a non-empty collection name when we get here
TRI_ASSERT(!coll.empty());
TRI_ASSERT(coll[0] < '0' || coll[0] > '9');
loadFromDB();
READ_LOCKER(guard, _authInfoLock);
auto it = _authInfo.find(username);

8
arangod/VocBase/AuthInfo.h
View File

@ -78,7 +78,7 @@ class AuthInfo {
_queryRegistry = registry;
}
/// Tells coordinator to reload his data. Only call in HearBeat thread
/// Tells coordinator to reload its data. Only called in HeartBeat thread
void outdate() { _outdated = true; }
/// Trigger eventual reload, user facing API call
@ -124,6 +124,12 @@ class AuthInfo {
std::string generateRawJwt(VPackBuilder const&);
private:
// internal method called by canUseCollection
// asserts that collection name is non-empty and already translated
// from collection id to name
AuthLevel canUseCollectionInternal(std::string const& username,
std::string const& dbname,
std::string const& coll);
void loadFromDB();
bool parseUsers(velocypack::Slice const& slice);
Result storeUserInternal(AuthUserEntry const& user, bool replace);

3
js/server/tests/recovery/corrupted-crc-mmfiles.js-disabled → js/server/tests/recovery/corrupted-crc-mmfiles.js
View File

@ -39,12 +39,15 @@ function runSetup () {
db._drop('UnitTestsRecovery');
var c = db._create('UnitTestsRecovery');
internal.wal.flush(true, true);
internal.wait(1, false);
internal.debugSetFailAt('BreakHeaderMarker');
for (var i = 0; i < 10; ++i) {
c.insert({ value: i });
}
internal.wal.flush(true, true);
internal.debugSetFailAt('LogfileManagerFlush');
internal.wait(5, false);
internal.debugSegfault('crashing server');
}

6
lib/Basics/debugging.cpp
View File

@ -277,7 +277,7 @@ void TRI_GetBacktrace(std::string& btstr) {
std::string(" [") + address + std::string("]\n");
}
TRI_SystemFree(symbol);
TRI_Free(symbol);
#else
void* stack_frames[50];
@ -318,7 +318,7 @@ void TRI_GetBacktrace(std::string& btstr) {
} else {
btstr += strings[i] + '\n';
}
TRI_SystemFree(demangled_name);
TRI_Free(demangled_name);
}
} else {
btstr += strings[i] + '\n';
@ -329,7 +329,7 @@ void TRI_GetBacktrace(std::string& btstr) {
}
}
if (strings != nullptr) {
TRI_SystemFree(strings);
TRI_Free(strings);
}
#endif
#endif

120
lib/Basics/memory.cpp
View File

@ -35,29 +35,6 @@
/// why so much memory is needed
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define MALLOC_WARNING_THRESHOLD (1024 * 1024 * 1024)
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief macros for producing stderr output
///
/// these will include the location of the problematic if we are in zone debug
/// mode, and will not include it if in non debug mode
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define ZONE_DEBUG_LOCATION " in %s:%d"
#define ZONE_DEBUG_PARAMS , file, line
#else
#define ZONE_DEBUG_LOCATION
#define ZONE_DEBUG_PARAMS
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief wrapper for malloc calls
////////////////////////////////////////////////////////////////////////////////
@ -86,16 +63,6 @@ static thread_local int AllowMemoryFailures = -1;
/// prints a warning if size is above a threshold
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
static inline void CheckSize(uint64_t n, char const* file, int line) {
// warn in the case of big malloc operations
if (n >= MALLOC_WARNING_THRESHOLD) {
fprintf(stderr, "big malloc action: %llu bytes" ZONE_DEBUG_LOCATION "\n",
(unsigned long long)n ZONE_DEBUG_PARAMS);
}
}
#endif
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
/// @brief timestamp for failing malloc
static inline double CurrentTimeStamp(void) {
@ -272,39 +239,9 @@ void operator delete[](void* pointer, std::nothrow_t const&) noexcept {
}
#endif
/// @brief system memory allocation
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void* TRI_SystemAllocateZ(uint64_t n, bool set, char const* file, int line) {
#else
void* TRI_SystemAllocate(uint64_t n, bool set) {
#endif
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
CheckSize(n, file, line);
#endif
char* m = static_cast<char*>(BuiltInMalloc((size_t)n));
if (m != nullptr) {
if (set) {
memset(m, 0, (size_t)n);
}
}
return m;
}
/// @brief basic memory management for allocate
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void* TRI_AllocateZ(uint64_t n,
char const* file, int line) {
#else
void* TRI_Allocate(uint64_t n) {
#endif
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
CheckSize(n, file, line);
#endif
char* m = static_cast<char*>(MALLOC_WRAPPER((size_t)n));
void* TRI_Allocate(size_t n) {
void* m = static_cast<char*>(MALLOC_WRAPPER(n));
if (m == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
@ -313,35 +250,19 @@ void* TRI_Allocate(uint64_t n) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
// prefill with 0xA5 (magic value, same as Valgrind will use)
memset(m, 0xA5, (size_t)n);
memset(m, 0xA5, n);
#endif
return m;
}
/// @brief basic memory management for reallocate
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void* TRI_ReallocateZ(void* m, uint64_t n,
char const* file, int line) {
#else
void* TRI_Reallocate(void* m, uint64_t n) {
#endif
void* TRI_Reallocate(void* m, size_t n) {
if (m == nullptr) {
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
return TRI_AllocateZ(n, file, line);
#else
return TRI_Allocate(n);
#endif
}
char* p = (char*)m;
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
CheckSize(n, file, line);
#endif
p = static_cast<char*>(REALLOC_WRAPPER(p, (size_t)n));
void* p = REALLOC_WRAPPER(m, n);
if (p == nullptr) {
TRI_set_errno(TRI_ERROR_OUT_OF_MEMORY);
@ -352,41 +273,16 @@ void* TRI_Reallocate(void* m, uint64_t n) {
}
/// @brief basic memory management for deallocate
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void TRI_FreeZ(void* m, char const* file, int line) {
#else
void TRI_Free(void* m) {
#endif
char* p = (char*)m;
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
if (p == nullptr) {
fprintf(stderr, "freeing nil ptr " ZONE_DEBUG_LOCATION ZONE_DEBUG_PARAMS);
if (m == nullptr) {
fprintf(stderr, "freeing nil ptr\n");
// crash intentionally
TRI_ASSERT(false);
}
#endif
free(p);
}
/// @brief free memory allocated by some low-level functions
///
/// this can be used to free memory that was not allocated by TRI_Allocate, but
/// by malloc et al
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void TRI_SystemFreeZ(void* p, char const* file, int line) {
#else
void TRI_SystemFree(void* p) {
#endif
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
if (p == nullptr) {
fprintf(stderr, "freeing nil ptr in %s:%d\n", file, line);
}
#endif
free(p);
free(m);
}
#ifdef ARANGODB_ENABLE_FAILURE_TESTS

54
lib/Basics/memory.h
View File

@ -28,73 +28,23 @@
#error use <Basics/Common.h>
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief system memory allocation
///
/// This will not add the memory zone information even when compiled with
/// --enable-maintainer-mode.
/// Internally, this will call just malloc, and probably memset.
/// Using this function instead of malloc/memset allows us to track all memory
/// allocations easier.
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_SystemAllocate(a, b) \
TRI_SystemAllocateZ((a), (b), __FILE__, __LINE__)
void* TRI_SystemAllocateZ(uint64_t, bool, char const*, int);
#else
void* TRI_SystemAllocate(uint64_t, bool);
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief basic memory management for allocate
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_Allocate(a) TRI_AllocateZ((a), __FILE__, __LINE__)
void* TRI_AllocateZ(uint64_t, char const*, int);
#else
void* TRI_Allocate(uint64_t);
#endif
void* TRI_Allocate(size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief basic memory management for reallocate
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_Reallocate(a, b) \
TRI_ReallocateZ((a), (b), __FILE__, __LINE__)
void* TRI_ReallocateZ(void*, uint64_t, char const*, int);
#else
void* TRI_Reallocate(void*, uint64_t);
#endif
void* TRI_Reallocate(void*, size_t);
////////////////////////////////////////////////////////////////////////////////
/// @brief basic memory management for deallocate
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_Free(a) TRI_FreeZ((a), __FILE__, __LINE__)
void TRI_FreeZ(void*, char const*, int);
#else
void TRI_Free(void*);
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief free memory allocated by low-level system functions
///
/// this can be used to free memory that was not allocated by TRI_Allocate, but
/// by system functions as malloc et al. This memory must not be passed to
/// TRI_Free because TRI_Free might subtract the memory zone from the original
/// pointer if compiled with --enable-maintainer-mode.
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_SystemFree(a) TRI_SystemFreeZ((a), __FILE__, __LINE__)
void TRI_SystemFreeZ(void*, char const*, int);
#else
void TRI_SystemFree(void*);
#endif
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
void TRI_AllowMemoryFailures();

11
lib/Basics/tri-strings.cpp
View File

@ -564,21 +564,10 @@ char* TRI_Concatenate3String(char const* a,
/// @brief frees a string
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
void TRI_FreeStringZ(char* value, char const* file,
int line) {
TRI_FreeZ(value, file, line);
}
#else
void TRI_FreeString(char* value) {
TRI_Free(value);
}
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief converts into hex representation
////////////////////////////////////////////////////////////////////////////////

9
lib/Basics/tri-strings.h
View File

@ -118,21 +118,16 @@ char* TRI_Concatenate3String(char const*, char const*,
/// @brief frees a string
////////////////////////////////////////////////////////////////////////////////
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
#define TRI_FreeString(a) TRI_FreeStringZ((a), __FILE__, __LINE__)
void TRI_FreeStringZ(char*, char const* file, int line);
#else
void TRI_FreeString(char*);
#endif
////////////////////////////////////////////////////////////////////////////////
/// @brief converts into hex reqpresentation
/// @brief converts into hex representation
////////////////////////////////////////////////////////////////////////////////
char* TRI_EncodeHexString(char const* source, size_t sourceLen, size_t* dstLen);
////////////////////////////////////////////////////////////////////////////////
/// @brief converts from hex reqpresentation
/// @brief converts from hex representation
////////////////////////////////////////////////////////////////////////////////
char* TRI_DecodeHexString(char const* source, size_t sourceLen, size_t* dstLen);

28
lib/Ssl/SslInterface.cpp
View File

@ -23,15 +23,16 @@
#include "SslInterface.h"
#include "Basics/Exceptions.h"
#include "Basics/StringUtils.h"
#include "Random/UniformCharacter.h"
#include <openssl/md5.h>
#include <openssl/sha.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <openssl/rand.h>
#include "Basics/StringUtils.h"
#include "Random/UniformCharacter.h"
#ifdef OPENSSL_NO_SSL2 // OpenSSL > 1.1.0 deprecates RAND_pseudo_bytes
#define RAND_BYTES RAND_bytes
#else
@ -217,7 +218,10 @@ void sslBASE64(char const* inputStr, char*& outputStr, size_t& outputLen) {
std::string sslPBKDF2HS1(char const* salt, size_t saltLength, char const* pass,
size_t passLength, int iter, int keyLength) {
unsigned char* dk =
(unsigned char*)TRI_SystemAllocate(keyLength + 1, false);
(unsigned char*)TRI_Allocate(keyLength + 1);
if (dk == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
PKCS5_PBKDF2_HMAC_SHA1(pass, (int)passLength, (const unsigned char*)salt,
(int)saltLength, iter, keyLength, dk);
@ -225,7 +229,7 @@ std::string sslPBKDF2HS1(char const* salt, size_t saltLength, char const* pass,
// return value as hex
std::string result =
StringUtils::encodeHex(std::string((char*)dk, keyLength));
TRI_SystemFree(dk);
TRI_Free(dk);
return result;
}
@ -250,7 +254,10 @@ std::string sslPBKDF2(char const* salt, size_t saltLength, char const* pass,
}
unsigned char* dk =
(unsigned char*)TRI_SystemAllocate(keyLength + 1, false);
(unsigned char*)TRI_Allocate(keyLength + 1);
if (dk == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
PKCS5_PBKDF2_HMAC(pass, (int)passLength, (const unsigned char*)salt,
(int)saltLength, iter, evp_md, keyLength, dk);
@ -258,7 +265,7 @@ std::string sslPBKDF2(char const* salt, size_t saltLength, char const* pass,
// return value as hex
std::string result =
StringUtils::encodeHex(std::string((char*)dk, keyLength));
TRI_SystemFree(dk);
TRI_Free(dk);
return result;
}
@ -283,14 +290,17 @@ std::string sslHMAC(char const* key, size_t keyLength, char const* message,
}
unsigned char* md =
(unsigned char*)TRI_SystemAllocate(EVP_MAX_MD_SIZE + 1, false);
(unsigned char*)TRI_Allocate(EVP_MAX_MD_SIZE + 1);
if (md == nullptr) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}
unsigned int md_len;
HMAC(evp_md, key, (int)keyLength, (const unsigned char*)message, messageLen,
md, &md_len);
std::string result = std::string((char*)md, md_len);
TRI_SystemFree(md);
TRI_Free(md);
return result;
}