From b7961ef3dd5ddbaa19f791fb89946bfc28b8440a Mon Sep 17 00:00:00 2001 From: Frank Celler Date: Fri, 26 May 2017 20:53:23 +0200 Subject: [PATCH] Reenabled snappy (#2504) * fixed snappy config * added numactl and GLIBCXX_FORCE_NEW * enabled snappy * fix snappy include path * windows paths --- 3rdParty/rocksdb/CMakeLists.txt | 18 +++-- 3rdParty/rocksdb/v5.1.4/thirdparty.inc | 8 +-- CMakeLists.txt | 2 + arangod/RocksDBEngine/RocksDBEngine.cpp | 30 ++------ cmake/FindSnappy.cmake | 91 ------------------------- cmake/Findsnappy.cmake | 1 + lib/CMakeLists.txt | 4 +- scripts/unittest | 9 ++- 8 files changed, 32 insertions(+), 131 deletions(-) delete mode 100644 cmake/FindSnappy.cmake create mode 100644 cmake/Findsnappy.cmake diff --git a/3rdParty/rocksdb/CMakeLists.txt b/3rdParty/rocksdb/CMakeLists.txt index 2e2472b6c5..17545defef 100644 --- a/3rdParty/rocksdb/CMakeLists.txt +++ b/3rdParty/rocksdb/CMakeLists.txt @@ -22,14 +22,18 @@ else () endif () # snappy settings -set(USE_SNAPPY_DEFAULT 1 CACHE BOOL "enable snappy") +set(SNAPPY_HOME ${CMAKE_SOURCE_DIR}/3rdParty/snappy/google-snappy-d53de18/) -set(SNAPPY_HOME ${CMAKE_SOURCE_DIR}/3rdParty/snappy/google-snappy-d53de18/ ) - -set(SNAPPY_INCLUDE ${SNAPPY_HOME} CACHE PATH "include path") -# list(APPEND SNAPPY_INCLUDE ${CMAKE_BINARY_DIR}/3rdParty/snappy/google-snappy-d53de18/) -set(SNAPPY_LIB_DEBUG snappystatic CACHE FILEPATH "debug library") -set(SNAPPY_LIB_RELEASE snappystatic CACHE FILEPATH "release library") +if (WIN32) + set(SNAPPY 1 CACHE BOOL "enable snappy") + set(SNAPPY_INCLUDE ${SNAPPY_HOME};${CMAKE_BINARY_DIR}/3rdParty/snappy/google-snappy-d53de18/) + set(SNAPPY_LIB_DEBUG snappystatic) + set(SNAPPY_LIB_RELEASE snappystatic) +else () + set(WITH_SNAPPY ON CACHE BOOL "enable snappy") + set(SNAPPY_INCLUDE_DIR ${SNAPPY_HOME};${CMAKE_BINARY_DIR}/3rdParty/snappy/google-snappy-d53de18/) + set(SNAPPY_LIBRARIES ${CMAKE_BINARY_DIR}/3rdParty/snappy/google-snappy-d53de18/libsnappystatic.a) +endif () add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/v5.1.4) diff --git a/3rdParty/rocksdb/v5.1.4/thirdparty.inc b/3rdParty/rocksdb/v5.1.4/thirdparty.inc index 9fffd9bff0..bab768273e 100644 --- a/3rdParty/rocksdb/v5.1.4/thirdparty.inc +++ b/3rdParty/rocksdb/v5.1.4/thirdparty.inc @@ -65,10 +65,10 @@ endif () # # Edit these 4 lines to define paths to Snappy # -set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library) -set(SNAPPY_INCLUDE ${SNAPPY_HOME}/inc/inc) -set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/bin/debug/amd64/snappy.lib) -set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/bin/retail/amd64/snappy.lib) +#set(SNAPPY_HOME $ENV{THIRDPARTY_HOME}/Snappy.Library) +#set(SNAPPY_INCLUDE ${SNAPPY_HOME}/inc/inc) +#set(SNAPPY_LIB_DEBUG ${SNAPPY_HOME}/bin/debug/amd64/snappy.lib) +#set(SNAPPY_LIB_RELEASE ${SNAPPY_HOME}/bin/retail/amd64/snappy.lib) # # Don't touch these lines diff --git a/CMakeLists.txt b/CMakeLists.txt index 9be801175f..473ac00513 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -611,6 +611,8 @@ include_directories(${ROCKSDB_INCLUDE_DIR}) set(ROCKSDB_LIBS rocksdblib;snappystatic) +add_dependencies(rocksdblib snappystatic) + if (USE_JEMALLOC) add_dependencies(rocksdblib jemalloc) endif () diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index b273feab20..b5b4c7e79f 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -222,13 +222,14 @@ void RocksDBEngine::start() { static_cast(opts->_maxBackgroundCompactions); _options.max_background_flushes = static_cast(opts->_maxFlushes); _options.use_fsync = opts->_useFSync; -/* + // only compress levels >= 2 _options.compression_per_level.resize(_options.num_levels); - for (size_t level = 0; i level < _options.num_levels; ++level) { - _options.compression_per_level[level] = ((level >= 2) ? rocksdb::kLZ4Compression : rocksdb::kNoCompression); + for (int level = 0; level < _options.num_levels; ++level) { + _options.compression_per_level[level] = ((level >= 2) ? rocksdb::kSnappyCompression : rocksdb::kNoCompression); } + // TODO: try out the effects of these options // Number of files to trigger level-0 compaction. A value <0 means that // level-0 compaction will not be triggered by number of files at all. // @@ -242,29 +243,6 @@ void RocksDBEngine::start() { // Maximum number of level-0 files. We stop writes at this point. _options.level0_stop_writes_trigger = 256; -*/ - - /* TODO: needs compile support for Snappy - // only compress levels >= 2 - _options.compression_per_level.resize(_options.num_levels); - for (int level = 0; level < _options.num_levels; ++level) { - _options.compression_per_level[level] = ((level >= 2) ? rocksdb::kSnappyCompression : rocksdb::kNoCompression); - } - */ - - /* TODO: try out the effects of these options - // Number of files to trigger level-0 compaction. A value <0 means that - // level-0 compaction will not be triggered by number of files at all. - _options.level0_file_num_compaction_trigger = -1; - - // Soft limit on number of level-0 files. We start slowing down writes at this - // point. A value <0 means that no writing slow down will be triggered by - // number of files in level-0. - _options.level0_slowdown_writes_trigger = -1; - - // Maximum number of level-0 files. We stop writes at this point. - _options.level0_stop_writes_trigger = 256; - */ _options.recycle_log_file_num = static_cast(opts->_recycleLogFileNum); _options.compaction_readahead_size = diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake deleted file mode 100644 index 32c9387694..0000000000 --- a/cmake/FindSnappy.cmake +++ /dev/null @@ -1,91 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find Snappy headers and libraries. -# -# Usage of this module as follows: -# -# find_package(Snappy) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Snappy_HOME - When set, this path is inspected instead of standard library -# locations as the root of the Snappy installation. -# The environment variable SNAPPY_HOME overrides this veriable. -# -# This module defines -# SNAPPY_INCLUDE_DIR, directory containing headers -# SNAPPY_LIBS, directory containing snappy libraries -# SNAPPY_STATIC_LIB, path to libsnappy.a -# SNAPPY_SHARED_LIB, path to libsnappy's shared library -# SNAPPY_FOUND, whether snappy has been found - -if( NOT "$ENV{SNAPPY_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "$ENV{SNAPPY_HOME}" _native_path ) - list( APPEND _snappy_roots ${_native_path} ) -elseif ( Snappy_HOME ) - list( APPEND _snappy_roots ${Snappy_HOME} ) -endif() - -# Try the parameterized roots, if they exist -if ( _snappy_roots ) - find_path( SNAPPY_INCLUDE_DIR NAMES snappy.h - PATHS ${_snappy_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( SNAPPY_LIBRARIES NAMES snappy - PATHS ${_snappy_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) -else () - find_path( SNAPPY_INCLUDE_DIR NAMES snappy.h ) - find_library( SNAPPY_LIBRARIES NAMES snappy ) -endif () - - -if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARIES) - set(SNAPPY_FOUND TRUE) - get_filename_component( SNAPPY_LIBS ${SNAPPY_LIBRARIES} PATH ) - set(SNAPPY_LIB_NAME libsnappy) - set(SNAPPY_STATIC_LIB ${SNAPPY_LIBS}/${SNAPPY_LIB_NAME}.a) - set(SNAPPY_SHARED_LIB ${SNAPPY_LIBS}/${SNAPPY_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(SNAPPY_FOUND FALSE) -endif () - -if (SNAPPY_FOUND) - if (NOT Snappy_FIND_QUIETLY) - message(STATUS "Found the Snappy library: ${SNAPPY_LIBRARIES}") - endif () -else () - if (NOT Snappy_FIND_QUIETLY) - set(SNAPPY_ERR_MSG "Could not find the Snappy library. Looked in ") - if ( _snappy_roots ) - set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} in ${_snappy_roots}.") - else () - set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} system search paths.") - endif () - if (Snappy_FIND_REQUIRED) - message(FATAL_ERROR "${SNAPPY_ERR_MSG}") - else (Snappy_FIND_REQUIRED) - message(STATUS "${SNAPPY_ERR_MSG}") - endif (Snappy_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - SNAPPY_INCLUDE_DIR - SNAPPY_LIBS - SNAPPY_LIBRARIES - SNAPPY_STATIC_LIB - SNAPPY_SHARED_LIB -) diff --git a/cmake/Findsnappy.cmake b/cmake/Findsnappy.cmake new file mode 100644 index 0000000000..dc252d5be1 --- /dev/null +++ b/cmake/Findsnappy.cmake @@ -0,0 +1 @@ +message(STATUS "using SNAPPY include path ${SNAPPY_INCLUDE}") diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index efd302f847..58b4cd53f4 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -89,10 +89,10 @@ if (ASM_OPTIMIZATIONS AND CMAKE_TARGET_ARCHITECTURE_CODE MATCHES "x86_64") endif() add_definitions("-DENABLE_ASM_CRC32=1") - message(STATUS "-- Will compile in hand-optimized assembler code for CRC32.") + message(STATUS "Will compile in hand-optimized assembler code for CRC32.") else() add_definitions("-DENABLE_ASM_CRC32=0") - message(STATUS "-- Will not compile in hand-optimized assembler code for CRC32.") + message(STATUS "Will not compile in hand-optimized assembler code for CRC32.") endif() if (USE_LOCAL_CLOCK_GETTIME) diff --git a/scripts/unittest b/scripts/unittest index 8f2bb8042b..b1322ba0ee 100755 --- a/scripts/unittest +++ b/scripts/unittest @@ -1,5 +1,6 @@ #!/bin/bash export PID=$$ +export GLIBCXX_FORCE_NEW=1 if test -n "$ORIGINAL_PATH"; then # running in cygwin... @@ -12,6 +13,12 @@ fi; export PORT=`expr 1024 + $RANDOM` +NUMA="" + +if `which numactl > /dev/null`; then + NUMA="numactl --interleave=all" +fi + if [ -z "${ARANGOSH}" ]; then if [ -x build/bin/arangosh ]; then ARANGOSH=build/bin/arangosh @@ -26,7 +33,7 @@ if [ -z "${ARANGOSH}" ]; then fi # -exec $ARANGOSH \ +exec $NUMA $ARANGOSH \ -c etc${PS}relative${PS}arangosh.conf \ --log.level warning \ --server.endpoint tcp://127.0.0.1:${PORT} \