From 50cb7b8a842b4683583471d1ad12f1e5ed97464a Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 8 May 2017 12:39:00 +0200 Subject: [PATCH 01/14] The rocksdb cluster dump now tests Fulltext index as well --- js/server/tests/dump/dump-rocksdb-cluster.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/js/server/tests/dump/dump-rocksdb-cluster.js b/js/server/tests/dump/dump-rocksdb-cluster.js index 743a913d82..443de35331 100644 --- a/js/server/tests/dump/dump-rocksdb-cluster.js +++ b/js/server/tests/dump/dump-rocksdb-cluster.js @@ -184,7 +184,7 @@ function dumpTestSuite () { assertEqual(2, c.type()); // document assertFalse(p.waitForSync); - assertEqual(7, c.getIndexes().length); + assertEqual(8, c.getIndexes().length); assertEqual("primary", c.getIndexes()[0].type); assertEqual("hash", c.getIndexes()[1].type); @@ -217,6 +217,10 @@ function dumpTestSuite () { assertTrue(c.getIndexes()[6].sparse); assertEqual([ "a_ss1", "a_ss2" ], c.getIndexes()[6].fields); + assertFalse(c.getIndexes()[7].unique); + assertEqual("fulltext", c.getIndexes()[7].type); + assertEqual([ "a_f" ], c.getIndexes()[7].fields); + assertEqual(0, c.count()); }, From 5cdaad53a8cd17a1257af6abc9c754ad5d2eae17 Mon Sep 17 00:00:00 2001 From: Simran Brucherseifer Date: Mon, 8 May 2017 14:14:38 +0200 Subject: [PATCH 02/14] Docs: Fix link between Manual and AQL book --- .../Books/Manual/Appendix/Deprecated/SimpleQueriesGeoQueries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Books/Manual/Appendix/Deprecated/SimpleQueriesGeoQueries.md b/Documentation/Books/Manual/Appendix/Deprecated/SimpleQueriesGeoQueries.md index 3d4eebff59..10471320b7 100644 --- a/Documentation/Books/Manual/Appendix/Deprecated/SimpleQueriesGeoQueries.md +++ b/Documentation/Books/Manual/Appendix/Deprecated/SimpleQueriesGeoQueries.md @@ -2,7 +2,7 @@ Geo Queries =========== {% hint 'warning' %} -It is recommended to use AQL instead, see [**Geo functions**](../../../../AQL/Functions/Geo.html). +It is recommended to use AQL instead, see [**Geo functions**](../../../AQL/Functions/Geo.html). {% endhint %} The ArangoDB allows to select documents based on geographic coordinates. In From 6474c952a2adade71536677a861763072d21d455 Mon Sep 17 00:00:00 2001 From: Andreas Streichardt Date: Mon, 8 May 2017 15:21:08 +0200 Subject: [PATCH 03/14] Windows warnings --- arangod/RocksDBEngine/RocksDBReplicationContext.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp index dedaa07ae7..bd0c436d88 100644 --- a/arangod/RocksDBEngine/RocksDBReplicationContext.cpp +++ b/arangod/RocksDBEngine/RocksDBReplicationContext.cpp @@ -184,7 +184,7 @@ RocksDBReplicationResult RocksDBReplicationContext::dump( while (_hasMore && buff.length() < chunkSize) { try { _hasMore = _iter->next(cb, 1); // TODO: adjust limit? - } catch (std::exception const& ex) { + } catch (std::exception const&) { _hasMore = false; return RocksDBReplicationResult(TRI_ERROR_INTERNAL, _lastTick); } catch (RocksDBReplicationResult const& ex) { @@ -244,7 +244,7 @@ arangodb::Result RocksDBReplicationContext::dumpKeyChunks(VPackBuilder& b, b.add("hash", VPackValue(std::to_string(hash))); b.close(); lowKey = ""; - } catch (std::exception const& ex) { + } catch (std::exception const&) { return Result(TRI_ERROR_INTERNAL); } } From 5400513087dfded27d5bad33d193e62f50ca781c Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 15:22:43 +0200 Subject: [PATCH 04/14] Possibly too ambitious timing --- js/client/tests/agency/agency-test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/client/tests/agency/agency-test.js b/js/client/tests/agency/agency-test.js index e4c30205a8..322c00103e 100644 --- a/js/client/tests/agency/agency-test.js +++ b/js/client/tests/agency/agency-test.js @@ -498,12 +498,12 @@ function agencyTestSuite () { assertEqual(readAndCheck([["/a/z"]]), [{"a":{"z":12}}]); writeAndCheck([[{"a/y":{"op":"set","new":12, "ttl": 1}}]]); assertEqual(readAndCheck([["/a/y"]]), [{"a":{"y":12}}]); - wait(1.0); + wait(1.1); assertEqual(readAndCheck([["/a/y"]]), [{a:{}}]); writeAndCheck([[{"/a/y":{"op":"set","new":12, "ttl": 1}}]]); writeAndCheck([[{"/a/y":{"op":"set","new":12}}]]); assertEqual(readAndCheck([["a/y"]]), [{"a":{"y":12}}]); - wait(1.0); + wait(1.1); assertEqual(readAndCheck([["/a/y"]]), [{"a":{"y":12}}]); writeAndCheck([[{"foo/bar":{"op":"set","new":{"baz":12}}}]]); assertEqual(readAndCheck([["/foo/bar/baz"]]), @@ -511,7 +511,7 @@ function agencyTestSuite () { assertEqual(readAndCheck([["/foo/bar"]]), [{"foo":{"bar":{"baz":12}}}]); assertEqual(readAndCheck([["/foo"]]), [{"foo":{"bar":{"baz":12}}}]); writeAndCheck([[{"foo/bar":{"op":"set","new":{"baz":12},"ttl":1}}]]); - wait(1.0); + wait(1.1); assertEqual(readAndCheck([["/foo"]]), [{"foo":{}}]); assertEqual(readAndCheck([["/foo/bar"]]), [{"foo":{}}]); assertEqual(readAndCheck([["/foo/bar/baz"]]), [{"foo":{}}]); From 61444b61547e118585a41da0a18c59126daf8dd6 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 15:46:45 +0200 Subject: [PATCH 05/14] Fixed header parsing in SimpleHttpClient for wrongly advanced cursor position in readBuffer --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index 62ed5e1733..46e6922287 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -635,6 +635,13 @@ void SimpleHttpClient::setRequest( // ----------------------------------------------------------------------------- void SimpleHttpClient::processHeader() { + if (_readBufferOffset > _readBuffer.length()) { + #warning Forgetmenot + + LOG_TOPIC(WARN, Logger::FIXME) + << _readBufferOffset << " " << _readBuffer.length() + << " " << std::string(_readBuffer.c_str(), _readBuffer.length()); + } TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); size_t remain = _readBuffer.length() - _readBufferOffset; char const* ptr = _readBuffer.c_str() + _readBufferOffset; @@ -658,12 +665,16 @@ void SimpleHttpClient::processHeader() { if (*ptr == '\r' || *ptr == '\n' || *ptr == '\0') { size_t len = pos - ptr; _readBufferOffset += len + 1; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + ptr += len + 1; remain -= len + 1; if (*pos == '\r') { // adjust offset if line ended with \r\n ++_readBufferOffset; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + ptr++; remain--; } @@ -729,6 +740,8 @@ void SimpleHttpClient::processHeader() { ptr += len + 1; _readBufferOffset += len + 1; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + remain -= (len + 1); TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); @@ -738,6 +751,8 @@ void SimpleHttpClient::processHeader() { if (pos == nullptr) { _readBufferOffset++; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + ptr++; remain--; } @@ -783,6 +798,8 @@ void SimpleHttpClient::processBody() { } _readBufferOffset += _result->getContentLength(); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + _result->setResultType(SimpleHttpResult::COMPLETE); _state = FINISHED; @@ -815,6 +832,8 @@ void SimpleHttpClient::processChunkedHeader() { // adjust offset if line ended with \r\n if (*pos == '\r') { ++_readBufferOffset; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + ++len; } @@ -894,6 +913,7 @@ void SimpleHttpClient::processChunkedBody() { } _readBufferOffset += (size_t)_nextChunkedSize + 2; + _state = IN_READ_CHUNKED_HEADER; processChunkedHeader(); } From 6d63f88464daac0e4777fc6ce5776df5b5503958 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 15:51:14 +0200 Subject: [PATCH 06/14] ooops left debug output behind --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index 62ed5e1733..cdac454ec6 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -658,12 +658,14 @@ void SimpleHttpClient::processHeader() { if (*ptr == '\r' || *ptr == '\n' || *ptr == '\0') { size_t len = pos - ptr; _readBufferOffset += len + 1; + ptr += len + 1; remain -= len + 1; if (*pos == '\r') { // adjust offset if line ended with \r\n ++_readBufferOffset; + ptr++; remain--; } @@ -729,18 +731,13 @@ void SimpleHttpClient::processHeader() { ptr += len + 1; _readBufferOffset += len + 1; + remain -= (len + 1); TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); TRI_ASSERT(ptr == _readBuffer.c_str() + _readBufferOffset); TRI_ASSERT(remain == _readBuffer.length() - _readBufferOffset); - pos = static_cast(memchr(ptr, '\n', remain)); - if (pos == nullptr) { - _readBufferOffset++; - ptr++; - remain--; - } } } } @@ -783,6 +780,7 @@ void SimpleHttpClient::processBody() { } _readBufferOffset += _result->getContentLength(); + _result->setResultType(SimpleHttpResult::COMPLETE); _state = FINISHED; @@ -894,6 +892,7 @@ void SimpleHttpClient::processChunkedBody() { } _readBufferOffset += (size_t)_nextChunkedSize + 2; + _state = IN_READ_CHUNKED_HEADER; processChunkedHeader(); } From 5b32a8162d765d3d508ab5d1e2f687a9f53c706e Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 15:56:35 +0200 Subject: [PATCH 07/14] oops more --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index 32f97f130e..704a94a8a2 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -665,7 +665,6 @@ void SimpleHttpClient::processHeader() { if (*ptr == '\r' || *ptr == '\n' || *ptr == '\0') { size_t len = pos - ptr; _readBufferOffset += len + 1; ->>>>>>> 61444b61547e118585a41da0a18c59126daf8dd6 ptr += len + 1; remain -= len + 1; @@ -673,7 +672,6 @@ void SimpleHttpClient::processHeader() { if (*pos == '\r') { // adjust offset if line ended with \r\n ++_readBufferOffset; ->>>>>>> 61444b61547e118585a41da0a18c59126daf8dd6 ptr++; remain--; From 5e5aa50bb7140a15fe92b286e8f20844617a1bad Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 8 May 2017 15:57:20 +0200 Subject: [PATCH 08/14] remove obsolete comment --- lib/Basics/socket-utils.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/Basics/socket-utils.cpp b/lib/Basics/socket-utils.cpp index 69be59356f..ac9d11ac03 100644 --- a/lib/Basics/socket-utils.cpp +++ b/lib/Basics/socket-utils.cpp @@ -51,19 +51,6 @@ int TRI_closesocket(TRI_socket_t s) { if (res != 0) { LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "socket close error: " << WSAGetLastError(); } - // We patch libev on Windows lightly to not really distinguish between - // socket handles and file descriptors, therefore, we do not have to do the - // following any more: - // if (s.fileDescriptor != -1) { - // res = _close(s.fileDescriptor); - // "To close a file opened with _open_osfhandle, call _close." - // The underlying handle is also closed by a call to _close, - // so it is not necessary to call the Win32 function CloseHandle - // on the original handle. - // However, we do want to do the special shutdown/recv magic above - // because only then we can reuse the port quickly, which we want - // to do directly after a port test. - // } } #else if (s.fileDescriptor != TRI_INVALID_SOCKET) { From 6c87bad730745dfccc9a96e9e7e45364502cd9e6 Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 15:59:26 +0200 Subject: [PATCH 09/14] fixing the fix --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index 704a94a8a2..f98ab84fe1 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -665,6 +665,7 @@ void SimpleHttpClient::processHeader() { if (*ptr == '\r' || *ptr == '\n' || *ptr == '\0') { size_t len = pos - ptr; _readBufferOffset += len + 1; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); ptr += len + 1; remain -= len + 1; @@ -672,6 +673,7 @@ void SimpleHttpClient::processHeader() { if (*pos == '\r') { // adjust offset if line ended with \r\n ++_readBufferOffset; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); ptr++; remain--; @@ -736,14 +738,17 @@ void SimpleHttpClient::processHeader() { ++len; } + // account for \n ptr += len + 1; _readBufferOffset += len + 1; + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); remain -= (len + 1); TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); TRI_ASSERT(ptr == _readBuffer.c_str() + _readBufferOffset); TRI_ASSERT(remain == _readBuffer.length() - _readBufferOffset); + pos = static_cast(memchr(ptr, '\n', remain)); } } @@ -787,6 +792,7 @@ void SimpleHttpClient::processBody() { } _readBufferOffset += _result->getContentLength(); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); _result->setResultType(SimpleHttpResult::COMPLETE); _state = FINISHED; From 37a6dd2a81dd3a4bba9721e7a52dc09e4e48d2a2 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 8 May 2017 15:59:33 +0200 Subject: [PATCH 10/14] remove unnecessary includes --- lib/Basics/memory.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/Basics/memory.cpp b/lib/Basics/memory.cpp index f9f85ded1e..270ea754e4 100644 --- a/lib/Basics/memory.cpp +++ b/lib/Basics/memory.cpp @@ -24,9 +24,6 @@ #include "Basics/Common.h" #ifdef ARANGODB_ENABLE_FAILURE_TESTS -#include -#include - #include #endif From 406bdeb3a1e44c15da66aca850f46a976f4c26fd Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 8 May 2017 16:00:45 +0200 Subject: [PATCH 11/14] fix indentation --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index f98ab84fe1..6f85b05cfd 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -665,7 +665,7 @@ void SimpleHttpClient::processHeader() { if (*ptr == '\r' || *ptr == '\n' || *ptr == '\0') { size_t len = pos - ptr; _readBufferOffset += len + 1; - TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); ptr += len + 1; remain -= len + 1; @@ -673,7 +673,7 @@ void SimpleHttpClient::processHeader() { if (*pos == '\r') { // adjust offset if line ended with \r\n ++_readBufferOffset; - TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); ptr++; remain--; @@ -741,7 +741,7 @@ void SimpleHttpClient::processHeader() { // account for \n ptr += len + 1; _readBufferOffset += len + 1; - TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); remain -= (len + 1); @@ -792,7 +792,7 @@ void SimpleHttpClient::processBody() { } _readBufferOffset += _result->getContentLength(); - TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); _result->setResultType(SimpleHttpResult::COMPLETE); _state = FINISHED; @@ -826,8 +826,7 @@ void SimpleHttpClient::processChunkedHeader() { // adjust offset if line ended with \r\n if (*pos == '\r') { ++_readBufferOffset; - TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); - + TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); ++len; } From 3d1a5893fc849e9333b9e8afe54a75e92d4ba74b Mon Sep 17 00:00:00 2001 From: Kaveh Vahedipour Date: Mon, 8 May 2017 16:13:05 +0200 Subject: [PATCH 12/14] fixing the fix --- lib/SimpleHttpClient/SimpleHttpClient.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/SimpleHttpClient/SimpleHttpClient.cpp b/lib/SimpleHttpClient/SimpleHttpClient.cpp index f98ab84fe1..d25d9a03ad 100644 --- a/lib/SimpleHttpClient/SimpleHttpClient.cpp +++ b/lib/SimpleHttpClient/SimpleHttpClient.cpp @@ -635,13 +635,6 @@ void SimpleHttpClient::setRequest( // ----------------------------------------------------------------------------- void SimpleHttpClient::processHeader() { - if (_readBufferOffset > _readBuffer.length()) { - #warning Forgetmenot - - LOG_TOPIC(WARN, Logger::FIXME) - << _readBufferOffset << " " << _readBuffer.length() - << " " << std::string(_readBuffer.c_str(), _readBuffer.length()); - } TRI_ASSERT(_readBufferOffset <= _readBuffer.length()); size_t remain = _readBuffer.length() - _readBufferOffset; char const* ptr = _readBuffer.c_str() + _readBufferOffset; From b40fc1074d02ef9df1f39e1d5ff2b14941aa3810 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 8 May 2017 17:26:40 +0200 Subject: [PATCH 13/14] try to detect situations where we need to invalidate the windows build directory --- Installation/Jenkins/build.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Installation/Jenkins/build.sh b/Installation/Jenkins/build.sh index 4dd49908eb..94dbc55260 100755 --- a/Installation/Jenkins/build.sh +++ b/Installation/Jenkins/build.sh @@ -653,13 +653,21 @@ fi PARTIAL_STATE=$? set -e +if test "${isCygwin}" == 1 -a "${PARTIAL_STATE}" == 0; then + # windows fails to partialy re-configure - so do a complete configure run. + if test -f CMakeFiles/generate.stamp -a CMakeFiles/generate.stamp -ot "${SOURCE_DIR}/CMakeList.txt"; then + echo "CMakeList older - Forcing complete configure run!" + PARTIAL_STATE=1 + fi +fi + if test "${PARTIAL_STATE}" == 0; then rm -rf CMakeFiles CMakeCache.txt CMakeCPackOptions.cmake cmake_install.cmake CPackConfig.cmake CPackSourceConfig.cmake CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" LDFLAGS="${LDFLAGS}" LIBS="${LIBS}" \ cmake ${SOURCE_DIR} ${CONFIGURE_OPTIONS} -G "${GENERATOR}" || exit 1 fi -if [ -n "$CPACK" -a -n "${TARGET_DIR}" -a -z "${MSVC}" ]; then +if [ -n "$CPACK" ] && [ -n "${TARGET_DIR}" ] && [ -z "${MSVC}" ]; then if ! grep -q CMAKE_STRIP CMakeCache.txt; then echo "cmake failed to detect strip; refusing to build unstripped packages!" exit 1 From 788fc07427ef84ba0a46d38d6f1ff56af0656ffa Mon Sep 17 00:00:00 2001 From: jsteemann Date: Mon, 8 May 2017 17:57:42 +0200 Subject: [PATCH 14/14] fix race --- arangod/Scheduler/ListenTask.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arangod/Scheduler/ListenTask.cpp b/arangod/Scheduler/ListenTask.cpp index bb3c61d771..edb9c4fd15 100644 --- a/arangod/Scheduler/ListenTask.cpp +++ b/arangod/Scheduler/ListenTask.cpp @@ -58,9 +58,13 @@ void ListenTask::start() { } catch (std::exception const& err) { LOG_TOPIC(WARN, arangodb::Logger::COMMUNICATION) << "failed to open endpoint '" << _endpoint->specification() << "' with error: " << err.what(); + return; } - _handler = [this](boost::system::error_code const& ec) { + TRI_ASSERT(_bound); + + auto self = shared_from_this(); + _handler = [this, self](boost::system::error_code const& ec) { // copy the shared_ptr so nobody can delete the Acceptor while the // callback is running std::shared_ptr acceptorCopy(_acceptor);