update gtest, use json output for reporting (#9193)

3rdParty/rocksdb/6.2/CMakeLists.txt

@@ -468,10 +468,10 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
-include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
 find_package(Threads REQUIRED)
 
-add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
+add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
 
 # Main library source code
 

3rdParty/rocksdb/6.2/Makefile

@@ -306,7 +306,7 @@ endif
 
 export GTEST_THROW_ON_FAILURE=1
 export GTEST_HAS_EXCEPTIONS=1
-GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
+GTEST_DIR = ./third-party/gtest-1.8.1/fused-src
 # AIX: pre-defined system headers are surrounded by an extern "C" block
 ifeq ($(PLATFORM), OS_AIX)
 	PLATFORM_CCFLAGS += -I$(GTEST_DIR)

3rdParty/rocksdb/6.2/src.mk

@@ -359,7 +359,7 @@ MAIN_SOURCES =                                                          \
   table/sst_file_reader_test.cc                                         \
   table/table_reader_bench.cc                                           \
   table/table_test.cc                                                   \
-  third-party/gtest-1.7.0/fused-src/gtest/gtest-all.cc                  \
+  third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc                  \
   tools/db_bench.cc                                                     \
   tools/db_bench_tool_test.cc                                           \
   tools/db_sanity_test.cc                                               \

3rdParty/rocksdb/6.2/third-party/gtest-1.8.1/fused-src/gtest/gtest_main.cc

@@ -0,0 +1,37 @@
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <stdio.h>
+#include "gtest/gtest.h"
+
+GTEST_API_ int main(int argc, char **argv) {
+  printf("Running main() from %s\n", __FILE__);
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

js/client/modules/@arangodb/testsuites/gtest.js

@@ -1,5 +1,5 @@
 /* jshint strict: false, sub: true */
-/* global */
+/* global print */
 'use strict';
 
 // //////////////////////////////////////////////////////////////////////////////
@@ -35,12 +35,16 @@ const optionsDocumentation = [
 
 const fs = require('fs');
 const pu = require('@arangodb/process-utils');
+const tu = require('@arangodb/test-utils');
 
 const testPaths = {
   'gtest': [],
   'catch': [],
 };
 
+const RED = require('internal').COLORS.COLOR_RED;
+const RESET = require('internal').COLORS.COLOR_RESET;
+
 // //////////////////////////////////////////////////////////////////////////////
 // / @brief TEST: GTest
 // //////////////////////////////////////////////////////////////////////////////
@@ -57,10 +61,62 @@ function locateGTest (name) {
   return file;
 }
 
+function readGreylist() {
+  let greylist = [];
+  const gtestGreylistRX = new RegExp('- gtest:.*', 'gm');
+  let raw_greylist = fs.read(fs.join('tests', 'Greylist.txt'));
+  let greylistMatches = raw_greylist.match(gtestGreylistRX);
+    if (greylistMatches != null) {
+    greylistMatches.forEach(function(match) {
+      let partMatch = /- gtest:(.*)/.exec(match);
+      if (partMatch.length !== 2) {
+        throw new Error("failed to match the test to greylist in: " + match);
+      }
+      greylist.push(partMatch[1]);
+    });
+  }
+  if (greylist.length !== 0) {
+    print(RED + "Greylisting tests: " + JSON.stringify(greylist) + RESET);
+  }
+  return greylist;
+}
+
+function getGTestResults(fileName, defaultResults) {
+  let results = defaultResults;
+  if (! fs.exists(fileName)) {
+    defaultResults.failed += 1;
+    print(RED + "No testresult file found at: " + fileName + RESET);    
+    return defaultResults;
+  }
+  let gTestResults = JSON.parse(fs.read(fileName));
+  results.failed = gTestResults.failures + gTestResults.errors;
+  results.status = (gTestResults.errors === 0) || (gTestResults.failures === 0);
+  gTestResults.testsuites.forEach(function(testSuite) {
+    results[testSuite.name] = {
+      failed: testSuite.failures + testSuite.errors,
+      status: (testSuite.failures + testSuite.errors ) === 0,
+      duration: testSuite.time
+    };
+    if (testSuite.failures !== 0) {
+      let message = "";
+      testSuite.testsuite.forEach(function (suite) {
+        if (suite.hasOwnProperty('failures')) {
+          suite.failures.forEach(function (fail) {
+            message += fail.failure;
+          });
+        }
+      });
+      results[testSuite.name].message = message;
+    }
+  });
+  return results;
+}
+
 function gtestRunner (options) {
   let results = { failed: 0 };
   let rootDir = fs.join(fs.getTempPath(), 'gtest');
-
+  let testResultJsonFile = fs.join(rootDir, 'testResults.json');
+  let greylist =   readGreylist();
   // we append one cleanup directory for the invoking logic...
   let dummyDir = fs.join(fs.getTempPath(), 'gtest_dummy');
   if (!fs.exists(dummyDir)) {
@@ -74,13 +130,18 @@ function gtestRunner (options) {
       let argv = [
         '--log.line-number',
         options.extremeVerbosity ? "true" : "false",
-        '--gtest_filter=-*_LongRunning'
+        '--gtest_output=json:' + testResultJsonFile,
+        '--gtest_filter=-*_LongRunning',
       ];
+      greylist.forEach(function(greyItem) {
+        argv.push('--gtest_filter=-'+greyItem);
+      });
       results.basics = pu.executeAndWait(run, argv, options, 'all-gtest', rootDir, false, options.coreCheck);
       results.basics.failed = results.basics.status ? 0 : 1;
       if (!results.basics.status) {
         results.failed += 1;
       }
+      results = getGTestResults(testResultJsonFile, results);
     } else {
       results.failed += 1;
       results.basics = {
@@ -90,7 +151,6 @@ function gtestRunner (options) {
       };
     }
   }
-
   return results;
 }
 

tests/CMakeLists.txt

@@ -232,7 +232,7 @@ target_include_directories(arangodbtests SYSTEM PRIVATE
 # the compiler will emit warnings for fakeit.hpp
 target_include_directories(arangodbtests SYSTEM PRIVATE
   ${CMAKE_SOURCE_DIR}/3rdParty/fakeit-gtest
-  ${CMAKE_SOURCE_DIR}/3rdParty/rocksdb/${ARANGO_ROCKSDB_VERSION}/third-party/gtest-1.7.0/fused-src
+  ${CMAKE_SOURCE_DIR}/3rdParty/rocksdb/${ARANGO_ROCKSDB_VERSION}/third-party/gtest-1.8.1/fused-src
 )
 
 find_package(OpenSSL REQUIRED)

tests/Greylist.txt

@@ -147,3 +147,10 @@ Mount: /unittest/paths
       [FAILED]  tests/js/server/recovery/corrupted-crc-mmfiles.js
 
       "test" failed: failed to read /work/tmp/arangosh_MNeeKL/crashtmp/6/tmp/testresult.json - ArangoError 2: No such file or directory: while reading /work/tmp/arangosh_MNeeKL/crashtmp/6/tmp/testresult.json
+
+======================================================================
+
+# - # gtest:IResearchLinkTest
+#
+# IResearchLinkTest.test_flush_marker - unreliable on windows under high load
+