diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index ae69a96b75..ae379ffbb8 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -266,7 +266,7 @@ while [ $# -gt 0 ]; do ;; --jemalloc) - CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DUSE_JEMALLOC=On" + USE_JEMALLOC=1 shift ;; @@ -318,6 +318,8 @@ while [ $# -gt 0 ]; do done + + if test -n "$LASTREV"; then lines=`git diff ${LASTREV}: ${COMPILE_MATTERS} | wc -l` @@ -341,6 +343,7 @@ elif [ "$CLANG36" == 1 ]; then CXX=/usr/bin/clang++-3.6 CXXFLAGS="${CXXFLAGS} -std=c++11" elif [ "${CXGCC}" = 1 ]; then + USE_JEMALLOC=0 if [ "${ARMV8}" = 1 ]; then export TOOL_PREFIX=aarch64-linux-gnu BUILD_DIR="${BUILD_DIR}-ARMV8" @@ -352,6 +355,7 @@ elif [ "${CXGCC}" = 1 ]; then exit 1; fi + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true" # -DCMAKE_LIBRARY_ARCHITECTURE=${TOOL_PREFIX} " export CXX=$TOOL_PREFIX-g++ export AR=$TOOL_PREFIX-ar export RANLIB=$TOOL_PREFIX-ranlib @@ -359,11 +363,20 @@ elif [ "${CXGCC}" = 1 ]; then export LD=$TOOL_PREFIX-g++ export LINK=$TOOL_PREFIX-g++ export STRIP=$TOOL_PREFIX-strip + + # we need ARM LD: + GOLD=0; + + # tell cmake we're cross compiling: CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DCROSS_COMPILING=true" + + # V8's mksnapshot won't work - ignore it: + MAKE_PARAMS="${MAKE_PARAMS} -i" fi - - +if [ "${USE_JEMALLOC}" = 1 ]; then + CONFIGURE_OPTIONS="${CONFIGURE_OPTIONS} -DUSE_JEMALLOC=On" +fi if [ "$SANITIZE" == 1 ]; then if [ "$GCC5" == 1 ]; then @@ -451,6 +464,7 @@ if [ ! -f Makefile -o ! -f CMakeCache.txt ]; then fi ${MAKE_CMD_PREFIX} ${MAKE} ${MAKE_PARAMS} + (cd ${SOURCE_DIR}; git rev-parse HEAD > last_compiled_version.sha) if [ -n "$CPACK" -a -n "${TARGET_DIR}" ]; then diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index da66c5d909..d3150af63c 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -546,7 +546,7 @@ static void UnsetOrKeep(arangodb::Transaction* trx, for (auto const& entry : VPackObjectIterator(value, false)) { TRI_ASSERT(entry.key.isString()); std::string key = entry.key.copyString(); - if (!((names.find(key) == names.end()) ^ unset)) { + if ((names.find(key) == names.end()) == unset) { // not found and unset or found and keep if (recursive && entry.value.isObject()) { result.add(entry.key); // Add the key diff --git a/arangod/Cluster/TraverserEngineRegistry.cpp b/arangod/Cluster/TraverserEngineRegistry.cpp index 8e19f738ba..398cbdc105 100644 --- a/arangod/Cluster/TraverserEngineRegistry.cpp +++ b/arangod/Cluster/TraverserEngineRegistry.cpp @@ -31,6 +31,7 @@ using namespace arangodb::traverser; +#ifndef USE_ENTERPRISE TraverserEngineRegistry::EngineInfo::EngineInfo(TRI_vocbase_t* vocbase, VPackSlice info) : _isInUse(false), @@ -40,6 +41,7 @@ TraverserEngineRegistry::EngineInfo::EngineInfo(TRI_vocbase_t* vocbase, TraverserEngineRegistry::EngineInfo::~EngineInfo() { } +#endif TraverserEngineRegistry::~TraverserEngineRegistry() { WRITE_LOCKER(writeLocker, _lock); diff --git a/arangod/VocBase/PathEnumerator.cpp b/arangod/VocBase/PathEnumerator.cpp index 00e333d1ae..df0d37fd21 100644 --- a/arangod/VocBase/PathEnumerator.cpp +++ b/arangod/VocBase/PathEnumerator.cpp @@ -53,10 +53,11 @@ bool DepthFirstEnumerator::next() { _edgeCursors.emplace(cursor); } } else { - TRI_ASSERT(!_enumeratedPath.edges.empty()); - // This path is at the end. cut the last step - _enumeratedPath.vertices.pop_back(); - _enumeratedPath.edges.pop_back(); + if (!_enumeratedPath.edges.empty()) { + // This path is at the end. cut the last step + _enumeratedPath.vertices.pop_back(); + _enumeratedPath.edges.pop_back(); + } } while (!_edgeCursors.empty()) { diff --git a/arangod/VocBase/datafile.cpp b/arangod/VocBase/datafile.cpp index 35f1afd052..37aabd0bac 100644 --- a/arangod/VocBase/datafile.cpp +++ b/arangod/VocBase/datafile.cpp @@ -44,6 +44,22 @@ using namespace arangodb::basics; namespace { +static std::string hexValue(uint64_t value) { + static const uint64_t Bits[] = { 56, 48, 40, 32, 24, 16, 8, 0 }; + + std::string line("0x"); + for (uint64_t i = 0; i < 8; ++i) { + uint8_t c = static_cast((static_cast(value) >> Bits[i]) & 0xFFULL); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + } + + return line; +} + /// @brief check if a marker appears to be created by ArangoDB 28 static TRI_voc_crc_t Crc28(TRI_voc_crc_t crc, void const* data, size_t length) { static TRI_voc_crc_t const CrcPolynomial = 0xEDB88320; @@ -445,7 +461,7 @@ char const* TRI_NameMarkerDatafile(TRI_df_marker_t const* marker) { return "abort transaction"; default: - return "unused/unknown"; + return "unknown"; } } @@ -472,7 +488,7 @@ bool TRI_IsValidMarkerDatafile(TRI_df_marker_t const* marker) { } if (marker->getSize() >= DatafileHelper::MaximalMarkerSize()) { - // a single marker bigger than 256 MB seems unreasonable + // a single marker bigger than this limit seems unreasonable // note: this is an arbitrary limit return false; } @@ -1184,8 +1200,9 @@ bool TRI_datafile_t::check(bool ignoreFailures) { // this function must not be called for non-physical datafiles TRI_ASSERT(isPhysical()); - char* ptr = _data; - char* end = _data + _currentSize; + char const* ptr = _data; + char const* end = _data + _currentSize; + char const* lastGood = nullptr; TRI_voc_size_t currentSize = 0; if (_currentSize == 0) { @@ -1200,7 +1217,7 @@ bool TRI_datafile_t::check(bool ignoreFailures) { [](TRI_voc_tick_t maxTick) -> void { TRI_UpdateTickServer(maxTick); }; while (ptr < end) { - TRI_df_marker_t* marker = reinterpret_cast(ptr); + TRI_df_marker_t const* marker = reinterpret_cast(ptr); TRI_voc_size_t const size = marker->getSize(); TRI_voc_tick_t const tick = marker->getTick(); TRI_df_marker_type_t const type = marker->getType(); @@ -1249,6 +1266,10 @@ bool TRI_datafile_t::check(bool ignoreFailures) { _state = TRI_DF_STATE_OPEN_ERROR; LOG(WARN) << "marker in datafile '" << getName() << "' points with size " << size << " beyond end of file"; + if (lastGood != nullptr) { + LOG(INFO) << "last good marker found at: " << hexValue(static_cast(static_cast(lastGood - _data))); + } + printMarker(marker, end - ptr, _data, end); updateTick(maxTick); @@ -1256,8 +1277,7 @@ bool TRI_datafile_t::check(bool ignoreFailures) { } // the following sanity check offers some, but not 100% crash-protection - // when reading - // totally corrupted datafiles + // when reading totally corrupted datafiles if (!TRI_IsValidMarkerDatafile(marker)) { if (type == 0 && size < 128) { // ignore markers with type 0 and a small size @@ -1273,6 +1293,10 @@ bool TRI_datafile_t::check(bool ignoreFailures) { _state = TRI_DF_STATE_OPEN_ERROR; LOG(WARN) << "marker in datafile '" << getName() << "' is corrupt: type: " << type << ", size: " << size; + if (lastGood != nullptr) { + LOG(INFO) << "last good marker found at: " << hexValue(static_cast(static_cast(lastGood - _data))); + } + printMarker(marker, size, _data, end); updateTick(maxTick); @@ -1347,70 +1371,11 @@ bool TRI_datafile_t::check(bool ignoreFailures) { LOG(WARN) << "crc mismatch found inside marker of type '" << TRI_NameMarkerDatafile(marker) << "' and size " << size << ". expected crc: " << CalculateCrcValue(marker) << ", actual crc: " << marker->getCrc(); - - { - LOG(INFO) << "raw marker data following:"; - char const* p = reinterpret_cast(marker); - char const* e = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); - if (e + 16 < end) { - // add some extra bytes for following data - e += 16; - } - - std::string line; - std::string raw; - size_t printed = 0; - while (p < e) { - // print offset - line.append("0x"); - static const uint64_t Bits[] = { 56, 48, 40, 32, 24, 16, 8, 0 }; - uint64_t offset = static_cast(static_cast(p - _data)); - for (uint64_t i = 0; i < 8; ++i) { - uint8_t c = static_cast((static_cast(offset) >> Bits[i]) & 0xFFULL); - uint8_t n1 = c >> 4; - uint8_t n2 = c & 0x0F; - - line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); - line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); - } - - // print data - line.append(": "); - for (size_t i = 0; i < 16; ++i) { - if (i == 8) { - // separate groups of 8 bytes - line.push_back(' '); - raw.push_back(' '); - } - - if (p >= e) { - line.append(" "); - } else { - uint8_t c = static_cast(*p++); - uint8_t n1 = c >> 4; - uint8_t n2 = c & 0x0F; - - line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); - line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); - line.push_back(' '); - - raw.push_back((c < 32 || c >= 127) ? '.' : static_cast(c)); - - ++printed; - } - } - - LOG(INFO) << line << " " << raw; - line.clear(); - raw.clear(); - - if (printed >= 2048) { - LOG(INFO) << "(output truncated due to excessive length)"; - break; - } - } + if (lastGood != nullptr) { + LOG(INFO) << "last good marker found at: " << hexValue(static_cast(static_cast(lastGood - _data))); } + printMarker(marker, size, _data, end); if (nextMarkerOk) { LOG(INFO) << "data directly following this marker looks ok so repairing the marker may recover it"; @@ -1443,6 +1408,7 @@ bool TRI_datafile_t::check(bool ignoreFailures) { return true; } + lastGood = ptr; ptr += alignedSize; } @@ -1450,6 +1416,62 @@ bool TRI_datafile_t::check(bool ignoreFailures) { return true; } +void TRI_datafile_t::printMarker(TRI_df_marker_t const* marker, TRI_voc_size_t size, char const* begin, char const* end) const { + LOG(INFO) << "raw marker data following:"; + LOG(INFO) << "type: " << TRI_NameMarkerDatafile(marker) << ", size: " << marker->getSize() << ", crc: " << marker->getCrc(); + LOG(INFO) << "(expected layout: size (4 bytes), crc (4 bytes), type and tick (8 bytes), payload following)"; + char const* p = reinterpret_cast(marker); + char const* e = reinterpret_cast(marker) + DatafileHelper::AlignedSize(size); + + if (e + 16 < end) { + // add some extra bytes for following data + e += 16; + } + + std::string line; + std::string raw; + size_t printed = 0; + while (p < e) { + // print offset + line.append(hexValue(static_cast(static_cast(p - begin)))); + + // print data + line.append(": "); + for (size_t i = 0; i < 16; ++i) { + if (i == 8) { + // separate groups of 8 bytes + line.push_back(' '); + raw.push_back(' '); + } + + if (p >= e) { + line.append(" "); + } else { + uint8_t c = static_cast(*p++); + uint8_t n1 = c >> 4; + uint8_t n2 = c & 0x0F; + + line.push_back((n1 < 10) ? ('0' + n1) : 'A' + n1 - 10); + line.push_back((n2 < 10) ? ('0' + n2) : 'A' + n2 - 10); + line.push_back(' '); + + raw.push_back((c < 32 || c >= 127) ? '.' : static_cast(c)); + + ++printed; + } + } + + LOG(INFO) << line << " " << raw; + line.clear(); + raw.clear(); + + if (printed >= 2048) { + LOG(INFO) << "(output truncated due to excessive length)"; + break; + } + } +} + /// @brief fixes a corrupted datafile bool TRI_datafile_t::fix(TRI_voc_size_t currentSize) { LOG(WARN) << "datafile '" << getName() << "' is corrupted at position " << currentSize; diff --git a/arangod/VocBase/datafile.h b/arangod/VocBase/datafile.h index 2d7c10c81d..e31823c847 100644 --- a/arangod/VocBase/datafile.h +++ b/arangod/VocBase/datafile.h @@ -263,6 +263,8 @@ struct TRI_datafile_t { /// @brief tries to repair a datafile bool tryRepair(); + void printMarker(TRI_df_marker_t const* marker, TRI_voc_size_t size, char const* begin, char const* end) const; + private: std::string _filename; // underlying filename TRI_voc_fid_t const _fid; // datafile identifier diff --git a/cmake/FindOpenSSL.cmake b/cmake/FindOpenSSL.cmake index 940ce796c9..01586506b7 100644 --- a/cmake/FindOpenSSL.cmake +++ b/cmake/FindOpenSSL.cmake @@ -56,7 +56,7 @@ if (UNIX) find_package(PkgConfig QUIET) - pkg_check_modules(_OPENSSL QUIET openssl) + pkg_check_modules(_OPENSSL openssl) endif () # Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES @@ -355,7 +355,6 @@ if(WIN32 AND NOT CYGWIN) set(OPENSSL_LIBRARIES ${SSL_EAY} ${LIB_EAY} ) endif() else() - find_library(OPENSSL_SSL_LIBRARY NAMES ssl diff --git a/cmake/packages/rpm.cmake b/cmake/packages/rpm.cmake index dd149933d0..13123453b0 100644 --- a/cmake/packages/rpm.cmake +++ b/cmake/packages/rpm.cmake @@ -32,14 +32,9 @@ set(CPACK_RPM_PACKAGE_RELOCATABLE FALSE) set(CPACK_TEMPORARY_DIRECTORY "${PROJECT_BINARY_DIR}/_CPack_Packages/${CMAKE_SYSTEM_NAME}/RPM/RPMS/${ARANGODB_PACKAGE_ARCHITECTURE}") set(CPACK_TEMPORARY_PACKAGE_FILE_NAME "${CPACK_TEMPORARY_DIRECTORY}/${CPACK_PACKAGE_FILE_NAME}.rpm") -get_cmake_property(_variableNames VARIABLES) -foreach (_variableName ${_variableNames}) - message(STATUS "${_variableName}=${${_variableName}}") -endforeach() - add_custom_target(package-arongodb-server COMMAND ${CMAKE_COMMAND} . - COMMAND ${CMAKE_CPACK_COMMAND} -V -G RPM + COMMAND ${CMAKE_CPACK_COMMAND} -G RPM COMMAND cp "${CPACK_TEMPORARY_DIRECTORY}/${CPACK_CLIENT_PACKAGE_FILE_NAME}.rpm" "${PROJECT_BINARY_DIR}" WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) list(APPEND PACKAGES_LIST package-arongodb-server)