Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix handling of comments in config files (#9173)

This commit is contained in:
Jan 2019-06-04 14:45:59 +02:00 committed by GitHub
parent 05559915b4
commit 036e966a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 286 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CHANGELOG
View File

@ -1,6 +1,20 @@
v3.3.24 (XXXX-XX-XX)
--------------------
* Fixed parsing of ArangoDB config files with inlined comments. Previous versions didn't handle
line comments properly if they were appended to an otherwise valid option value.
For example, the comment in the line
max-total-wal-size = 1024000 # 1M
was not ignored and made part of the value. In the end, the value was interpreted as if
max-total-wal-size = 10240001000000
was specified.
This version fixes the handling of comments in the config files so that they behave as intended.
* fixed a crash when posting an async request to the server using the "x-arango-async"
request header and the server's request queue was full
@ -10,6 +24,7 @@ v3.3.24 (XXXX-XX-XX)
* fix agency issue in abort of cleanOutServer job
v3.3.23 (2019-04-14)
--------------------

1
lib/CMakeLists.txt
View File

@ -203,6 +203,7 @@ add_library(${LIB_ARANGO} STATIC
Maskings/Path.cpp
Maskings/RandomStringMask.cpp
ProgramOptions/Option.cpp
ProgramOptions/Parameters.cpp
ProgramOptions/ProgramOptions.cpp
ProgramOptions/Section.cpp
ProgramOptions/Translator.cpp

40
lib/ProgramOptions/Parameters.cpp Normal file
View File

@ -0,0 +1,40 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2016 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Jan Steemann
////////////////////////////////////////////////////////////////////////////////
#include "ProgramOptions/Parameters.h"
#include <regex>
namespace {
std::regex const removeComments("(^[ \t]+|[ \t]*(#.*)?$)", std::regex::nosubs | std::regex::ECMAScript);
}
namespace arangodb {
namespace options {
std::string removeCommentsFromNumber(std::string const& value) {
// replace leading spaces, replace trailing spaces & comments
return std::regex_replace(value, ::removeComments, "");
}
}
}

26
lib/ProgramOptions/Parameters.h
View File

@ -34,14 +34,22 @@
#include <limits>
#include <numeric>
#include <type_traits>
#include <iostream>
namespace arangodb {
namespace options {
// helper functions to strip-non-numeric data from a string
std::string removeCommentsFromNumber(std::string const& value);
// convert a string into a number, base version for signed integer types
template <typename T>
inline typename std::enable_if<std::is_signed<T>::value, T>::type toNumber(std::string value,
T base) {
// replace leading spaces, replace trailing spaces & comments
value = removeCommentsFromNumber(value);
auto n = value.size();
int64_t m = 1;
int64_t d = 1;
@ -51,15 +59,15 @@ inline typename std::enable_if<std::is_signed<T>::value, T>::type toNumber(std::
if (suffix == "kib" || suffix == "KiB") {
m = 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
} else if (suffix == "mib" || suffix == "MiB") {
m = 1024 * 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
} else if (suffix == "gib" || suffix == "GiB") {
m = 1024 * 1024 * 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
}
}
@ -98,6 +106,7 @@ inline typename std::enable_if<std::is_signed<T>::value, T>::type toNumber(std::
value = value.substr(0, n - 1);
}
}
auto v = static_cast<int64_t>(std::stoll(value));
if (v < static_cast<int64_t>((std::numeric_limits<T>::min)()) ||
v > static_cast<int64_t>((std::numeric_limits<T>::max)())) {
@ -110,6 +119,9 @@ inline typename std::enable_if<std::is_signed<T>::value, T>::type toNumber(std::
template <typename T>
inline typename std::enable_if<std::is_unsigned<T>::value, T>::type toNumber(std::string value,
T base) {
// replace leading spaces, replace trailing spaces & comments
value = removeCommentsFromNumber(value);
auto n = value.size();
uint64_t m = 1;
uint64_t d = 1;
@ -119,15 +131,15 @@ inline typename std::enable_if<std::is_unsigned<T>::value, T>::type toNumber(std
if (suffix == "kib" || suffix == "KiB") {
m = 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
} else if (suffix == "mib" || suffix == "MiB") {
m = 1024 * 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
} else if (suffix == "gib" || suffix == "GiB") {
m = 1024 * 1024 * 1024;
value = value.substr(0, n - 2);
value = value.substr(0, n - 3);
seen = true;
}
}
@ -177,7 +189,7 @@ inline typename std::enable_if<std::is_unsigned<T>::value, T>::type toNumber(std
// convert a string into a number, version for double values
template <>
inline double toNumber<double>(std::string value, double base) {
return std::stod(value);
return std::stod(removeCommentsFromNumber(value));
}
// convert a string into another type, specialized version for numbers

210
tests/Basics/InifileParserTest.cpp Normal file
View File

@ -0,0 +1,210 @@
////////////////////////////////////////////////////////////////////////////////
/// @brief test suite for StringUtils class
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2004-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
/// @author Copyright 2007-2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////
#include "Basics/Common.h"
#include "catch.hpp"
#include <iomanip>
#include <sstream>
#include "Basics/files.h"
#include "Basics/FileUtils.h"
#include "Basics/StringUtils.h"
#include "Logger/Logger.h"
#include "ProgramOptions/IniFileParser.h"
#include "ProgramOptions/Parameters.h"
#include "ProgramOptions/ProgramOptions.h"
#include "Random/RandomGenerator.h"
using namespace arangodb;
using namespace arangodb::basics;
struct IniFilesSetup {
IniFilesSetup () {
long systemError;
std::string errorMessage;
_directory.append(TRI_GetTempPath());
_directory.push_back(TRI_DIR_SEPARATOR_CHAR);
_directory.append("arangotest-");
_directory.append(std::to_string(static_cast<uint64_t>(TRI_microtime())));
_directory.append(std::to_string(arangodb::RandomGenerator::interval(UINT32_MAX)));
TRI_CreateDirectory(_directory.c_str(), systemError, errorMessage);
}
~IniFilesSetup () {
// let's be sure we delete the right stuff
TRI_ASSERT(_directory.length() > 10);
TRI_RemoveDirectory(_directory.c_str());
}
std::string _directory;
};
TEST_CASE("IniFileParserTest", "[ini]") {
IniFilesSetup s;
////////////////////////////////////////////////////////////////////////////////
/// @brief test_parsing
////////////////////////////////////////////////////////////////////////////////
SECTION("test_parsing") {
using namespace arangodb::options;
uint64_t writeBufferSize = UINT64_MAX;
uint64_t totalWriteBufferSize = UINT64_MAX;
uint64_t maxWriteBufferNumber = UINT64_MAX;
uint64_t maxTotalWalSize = UINT64_MAX;
uint64_t blockCacheSize = UINT64_MAX;
bool enforceBlockCacheSizeLimit = false;
uint64_t cacheSize = UINT64_MAX;
uint64_t nonoSetOption = UINT64_MAX;
uint64_t someValueUsingSuffixes = UINT64_MAX;
uint64_t someOtherValueUsingSuffixes = UINT64_MAX;
uint64_t yetSomeOtherValueUsingSuffixes = UINT64_MAX;
uint64_t andAnotherValueUsingSuffixes = UINT64_MAX;
uint64_t andFinallySomeGb = UINT64_MAX;
uint64_t aValueWithAnInlineComment = UINT64_MAX;
bool aBoolean = false;
bool aBooleanTrue = false;
bool aBooleanFalse = true;
bool aBooleanNotSet = false;
double aDouble = -2.0;
double aDoubleWithAComment = -2.0;
double aDoubleNotSet = -2.0;
std::string aStringValueEmpty = "snort";
std::string aStringValue = "purr";
std::string aStringValueWithAnInlineComment = "gaw";
std::string anotherStringValueWithAnInlineComment = "gaw";
std::string aStringValueNotSet = "meow";
ProgramOptions options("testi", "testi [options]", "bla", "/tmp/bla");
options.addSection("rocksdb", "bla");
options.addOption("--rocksdb.write-buffer-size", "bla", new UInt64Parameter(&writeBufferSize));
options.addOption("--rocksdb.total-write-buffer-size", "bla", new UInt64Parameter(&totalWriteBufferSize));
options.addOption("--rocksdb.max-write-buffer-number", "bla", new UInt64Parameter(&maxWriteBufferNumber));
options.addOption("--rocksdb.max-total-wal-size", "bla", new UInt64Parameter(&maxTotalWalSize));
options.addOption("--rocksdb.block-cache-size", "bla", new UInt64Parameter(&blockCacheSize));
options.addOption("--rocksdb.enforce-block-cache-size-limit", "bla", new BooleanParameter(&enforceBlockCacheSizeLimit));
options.addSection("cache", "bla");
options.addOption("--cache.size", "bla", new UInt64Parameter(&cacheSize));
options.addOption("--cache.nono-set-option", "bla", new UInt64Parameter(&nonoSetOption));
options.addSection("pork", "bla");
options.addOption("--pork.a-boolean", "bla", new BooleanParameter(&aBoolean, true));
options.addOption("--pork.a-boolean-true", "bla", new BooleanParameter(&aBooleanTrue, true));
options.addOption("--pork.a-boolean-false", "bla", new BooleanParameter(&aBooleanFalse, true));
options.addOption("--pork.a-boolean-not-set", "bla", new BooleanParameter(&aBooleanNotSet, true));
options.addOption("--pork.some-value-using-suffixes", "bla", new UInt64Parameter(&someValueUsingSuffixes));
options.addOption("--pork.some-other-value-using-suffixes", "bla", new UInt64Parameter(&someOtherValueUsingSuffixes));
options.addOption("--pork.yet-some-other-value-using-suffixes", "bla", new UInt64Parameter(&yetSomeOtherValueUsingSuffixes));
options.addOption("--pork.and-another-value-using-suffixes", "bla", new UInt64Parameter(&andAnotherValueUsingSuffixes));
options.addOption("--pork.and-finally-some-gb", "bla", new UInt64Parameter(&andFinallySomeGb));
options.addOption("--pork.a-value-with-an-inline-comment", "bla", new UInt64Parameter(&aValueWithAnInlineComment));
options.addOption("--pork.a-double", "bla", new DoubleParameter(&aDouble));
options.addOption("--pork.a-double-with-a-comment", "bla", new DoubleParameter(&aDoubleWithAComment));
options.addOption("--pork.a-double-not-set", "bla", new DoubleParameter(&aDoubleNotSet));
options.addOption("--pork.a-string-value-empty", "bla", new StringParameter(&aStringValueEmpty));
options.addOption("--pork.a-string-value", "bla", new StringParameter(&aStringValue));
options.addOption("--pork.a-string-value-with-an-inline-comment", "bla", new StringParameter(&aStringValueWithAnInlineComment));
options.addOption("--pork.another-string-value-with-an-inline-comment", "bla", new StringParameter(&anotherStringValueWithAnInlineComment));
options.addOption("--pork.a-string-value-not-set", "bla", new StringParameter(&aStringValueNotSet));
auto contents = R"data(
[rocksdb]
# Write buffers
write-buffer-size = 2048000 # 2M
total-write-buffer-size = 536870912
max-write-buffer-number = 4
max-total-wal-size = 1024000 # 1M
# Read buffers
block-cache-size = 268435456
enforce-block-cache-size-limit = true
[cache]
size = 268435456 # 256M
[pork]
a-boolean = true
a-boolean-true = true
a-boolean-false = false
some-value-using-suffixes = 1M
some-other-value-using-suffixes = 1MiB
yet-some-other-value-using-suffixes = 12MB
and-another-value-using-suffixes = 256kb
and-finally-some-gb = 256GB
a-value-with-an-inline-comment = 12345#1234M
a-double = 335.25
a-double-with-a-comment = 2948.434#343
a-string-value-empty =
a-string-value = 486hbsbq,r
a-string-value-with-an-inline-comment = abc#def h
another-string-value-with-an-inline-comment = abc #def h
)data";
// create a temp file with the above options
std::string filename = basics::FileUtils::buildFilename(s._directory, "testi.conf");
basics::FileUtils::spit(filename, contents);
IniFileParser parser(&options);
bool result = parser.parse(filename, true);
CHECK(true == result);
CHECK(2048000U == writeBufferSize);
CHECK(536870912U == totalWriteBufferSize);
CHECK(4U == maxWriteBufferNumber);
CHECK(1024000U == maxTotalWalSize);
CHECK(268435456U == blockCacheSize);
CHECK(true == enforceBlockCacheSizeLimit);
CHECK(268435456U == cacheSize);
CHECK(UINT64_MAX == nonoSetOption);
CHECK(true == aBoolean);
CHECK(true == aBooleanTrue);
CHECK(false == aBooleanFalse);
CHECK(false == aBooleanNotSet);
CHECK(1000000U == someValueUsingSuffixes);
CHECK(1048576U == someOtherValueUsingSuffixes);
CHECK(12000000U == yetSomeOtherValueUsingSuffixes);
CHECK(256000U == andAnotherValueUsingSuffixes);
CHECK(256000000000U == andFinallySomeGb);
CHECK(12345U == aValueWithAnInlineComment);
CHECK(335.25 == aDouble);
CHECK(2948.434 == aDoubleWithAComment);
CHECK(-2.0 == aDoubleNotSet);
CHECK("" == aStringValueEmpty);
CHECK("486hbsbq,r" == aStringValue);
CHECK("abc#def h" == aStringValueWithAnInlineComment);
CHECK("abc #def h" == anotherStringValueWithAnInlineComment);
CHECK("meow" == aStringValueNotSet);
}
}

1
tests/CMakeLists.txt
View File

@ -36,6 +36,7 @@ add_executable(
Basics/vector-test.cpp
Basics/structure-size-test.cpp
Basics/EndpointTest.cpp
Basics/InifileParserTest.cpp
Basics/StringBufferTest.cpp
Basics/StringUtilsTest.cpp
Basics/VelocyPackHelper-test.cpp