Feature/jenkins pipeline (#3218)

Installation/Pipeline/Jenkinsfile.groovy

@@ -202,7 +202,7 @@ def checkCoresAndSave(os, runDir, name, archRuns, archCores) {
         }
     }
     else {
-        sh "for i in logs out tmp; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"
+        sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"
 
         def files = findFiles(glob: '${runDir}/core*')
 
@@ -400,14 +400,11 @@ def checkCommitMessages() {
             ]
         }
         else {
-
             restrictions = [
                 "build-community-mac" : true,
-                // "build-community-windows" : true,
                 "build-enterprise-linux" : true,
                 "test-cluster-enterprise-rocksdb-linux" : true,
-                "test-singleserver-community-mmfiles-mac" : true
+                "test-singleserver-enterprise-mmfiles-linux" : true
-                // "test-singleserver-community-rocksdb-windows" : true
             ]
         }
     }
@@ -438,6 +435,10 @@ Restrictions: ${restrictions.keySet().join(", ")}
 def stashBinaries(os, edition) {
     def paths = ["build/etc", "etc", "Installation/Pipeline", "js", "scripts", "UnitTests"]
 
+    if (edition == "enterprise") {
+       paths << "enterprise/js"
+    }
+
     if (os == "windows") {
         paths << "build/bin/RelWithDebInfo"
         paths << "build/tests/RelWithDebInfo"
@@ -488,75 +489,68 @@ def jslint() {
 // -----------------------------------------------------------------------------
 
 def getTests(os, edition, mode, engine) {
+    def tests = [
+        ["arangobench", "arangobench" , ""],
+        ["arangosh", "arangosh", "--skipShebang true"],
+        ["authentication", "authentication", ""],
+        ["authentication_parameters", "authentication_parameters", ""],
+        ["config", "config" , ""],
+        ["dump", "dump" , ""],
+        ["dump_authentication", "dump_authentication" , ""],
+        ["endpoints", "endpoints", ""],
+        ["server_http", "server_http", ""],
+        ["shell_client", "shell_client", ""],
+        ["shell_server", "shell_server", ""],
+        ["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0", ,""],
+        ["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1", ,""],
+        ["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2", ,""],
+        ["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3", ,""],
+        ["upgrade", "upgrade" , ""],
+        rspecify(os, "http_server"),
+        rspecify(os, "ssl_server")
+    ]
+
+    if (edition == "enterprise") {
+        tests += [
+            ["authentication_server", "authentication_server", ""]
+        ]
+    }
+
     if (mode == "singleserver") {
-        return [
+        tests += [
             ["agency", "agency", ""],
             ["boost", "boost", "--skipCache false"],
-            ["arangobench", "arangobench", ""],
-            ["arangosh", "arangosh", "--skipShebang true"],
-            ["authentication", "authentication", ""],
-            ["authentication_server", "authentication_server", ""],
-            ["authentication_parameters", "authentication_parameters", ""],
             ["cluster_sync", "cluster_sync", ""],
-            ["config", "config", ""],
             ["dfdb", "dfdb", ""],
-            //"dump",
-            //"dump_authentication",
-            ["endpoints", "endpoints", ""],
-            rspecify(os, "http_replication"),
-            rspecify(os, "http_server"),
-            ["replication_sync", "replication_sync", ""],
-            ["replication_static", "replication_static", ""],
             ["replication_ongoing", "replication_ongoing", ""],
-            ["server_http", "server_http", ""],
+            ["replication_static", "replication_static", ""],
-            ["shell_client", "shell_client", ""],
+            ["replication_sync", "replication_sync", ""],
             ["shell_replication", "shell_replication", ""],
-            ["shell_server", "shell_server", ""],
+            rspecify(os, "http_replication")
-            ["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0", ,""],
-            ["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1", ,""],
-            ["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2", ,""],
-            ["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3", ,""],
-            rspecify(os, "ssl_server"),
-            ["upgrade", "upgrade" , ""]
-        ]
-    }
-    else {
-        return [
-            ["arangobench", "arangobench" , ""],
-            ["arangosh", "arangosh" , "--skipShebang true"],
-            ["authentication", "authentication" , ""],
-            ["authentication_server", "authentication_server", ""],
-            ["authentication_parameters", "authentication_parameters" , ""],
-            ["config", "config" , ""],
-            ["dump", "dump" , ""],
-            ["dump_authentication", "dump_authentication" , ""],
-            ["endpoints", "endpoints" , ""],
-            rspecify(os, "http_server"),
-            ["server_http", "server_http", ""],
-            ["shell_client", "shell_client", ""],
-            ["shell_server", "shell_server", ""],
-            ["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0"],
-            ["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1"],
-            ["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2"],
-            ["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3"],
-            rspecify(os, "ssl_server"),
-            ["upgrade", "upgrade" , ""]
         ]
     }
+
+    return tests
 }
 
-def setupTestEnvironment(os, logFile, runDir) {
+def setupTestEnvironment(os, edition, logFile, runDir) {
     fileOperations([
         folderCreateOperation("${runDir}/tmp"),
     ])
 
+    def subdirs = ['build', 'etc', 'js', 'UnitTests']
+
+    if (edition == "enterprise") {
+       subdirs << "enterprise"
+    }
+
     if (os == "windows") {
-        for (file in ['build', 'etc', 'js', 'UnitTests']) {
+        for (file in subdirs) {
             powershell "cd ${runDir} ; New-Item -Path ${file} -ItemType SymbolicLink -Value ..\\${file} | Out-Null"
         }
     }
     else {
-        for (file in ['build', 'etc', 'js', 'UnitTests']) {
+        for (file in subdirs) {
             sh "ln -s ../${file} ${runDir}/${file}"
         }
 
@@ -609,7 +603,7 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
 
             try {
                 lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${engine}-${lockIndex}") {
-                    setupTestEnvironment(os, logFile, runDir)
+                    setupTestEnvironment(os, edition, logFile, runDir)
 
                     try {
                         timeout(30) {
@@ -621,14 +615,16 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
                                     powershell "cd ${runDir} ; ${command} | Add-Content -PassThru ${logFile}"
                                 }
                                 else {
+                                    sh "echo \"Host: `hostname`\" | tee ${logFile}"
+                                    sh "echo \"PWD:  `pwd`\" | tee -a ${logFile}"
+                                    sh "echo \"Date: `date`\" | tee -a ${logFile}"
+
                                     command = "(cd ${runDir} ; echo 1 > result ; ${command} ; echo \$? > result) 2>&1 | " +
-                                              "tee ${logFile} ; exit `cat ${runDir}/result`"
+                                              "tee -a ${logFile} ; exit `cat ${runDir}/result`"
                                     echo "executing ${command}"
                                     sh command
                                 }
                             }
-
-                            checkCoresAndSave(os, runDir, name, archRuns, archCores)
                         }
                     }
                     catch (exc) {
@@ -637,6 +633,9 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
                         copyFile(os, logFile, logFileFailed)
                         throw exc
                     }
+                    finally {
+                        checkCoresAndSave(os, runDir, name, archRuns, archCores)
+                    }
                 }
             }
             catch (exc) {
@@ -673,62 +672,64 @@ def testCheck(os, edition, mode, engine) {
 
 def testStep(os, edition, mode, engine, testName) {
     return {
-        node(testJenkins[os]) {
+        if (testCheck(os, edition, mode, engine)) {
-            stage(testName) {
+            node(testJenkins[os]) {
-                def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
+                stage(testName) {
-                def archFailedRel = "${archRel}_FAILED"
+                    def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
-                def archRunsRel = "${archRel}_RUN"
+                    def archFailedRel = "${archRel}_FAILED"
-                def archCoresRel = "${archRel}_CORES"
+                    def archRunsRel = "${archRel}_RUN"
+                    def archCoresRel = "${archRel}_CORES"
 
-                def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
+                    def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
-                def archFailed = "${arch}_FAILED"
+                    def archFailed = "${arch}_FAILED"
-                def archRuns = "${arch}_RUN"
+                    def archRuns = "${arch}_RUN"
-                def archCores = "${arch}_CORES"
+                    def archCores = "${arch}_CORES"
 
-                // clean the current workspace completely
+                    // clean the current workspace completely
-                deleteDir()
+                    deleteDir()
 
-                // create directories for the artifacts
+                    // create directories for the artifacts
-                fileOperations([
+                    fileOperations([
-                    folderCreateOperation(arch),
+                        folderCreateOperation(arch),
-                    folderCreateOperation(archFailed),
+                        folderCreateOperation(archFailed),
-                    folderCreateOperation(archRuns),
+                        folderCreateOperation(archRuns),
-                    folderCreateOperation(archCores)
+                        folderCreateOperation(archCores)
-                ])
+                    ])
 
-                // unstash binaries
+                    // unstash binaries
-                unstashBinaries(os, edition)
+                    unstashBinaries(os, edition)
 
-                // find a suitable port
+                    // find a suitable port
-                def port = (getStartPort(os) as Integer)
+                    def port = (getStartPort(os) as Integer)
-                echo "Using start port: ${port}"
+                    echo "Using start port: ${port}"
 
-                // seriously...60 minutes is the super absolute max max max.
+                    // seriously...60 minutes is the super absolute max max max.
-                // even in the worst situations ArangoDB MUST be able to finish within 60 minutes
+                    // even in the worst situations ArangoDB MUST be able to finish within 60 minutes
-                // even if the features are green this is completely broken performance wise..
+                    // even if the features are green this is completely broken performance wise..
-                // DO NOT INCREASE!!
+                    // DO NOT INCREASE!!
 
-                timeout(60) {
+                    timeout(60) {
-                    try {
+                        try {
-                        executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
+                            executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
-                    }
-                    finally {
-                        // step([$class: 'XUnitBuilder',
-                        //     thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
-                        //     tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])
-
-                        // release the port reservation
-                        if (os == 'linux' || os == 'mac') {
-                            sh "Installation/Pipeline/port.sh --clean ${port}"
-                        }
-                        else if (os == 'windows') {
-                            powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
                         }
+                        finally {
+                            // step([$class: 'XUnitBuilder',
+                            //     thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
+                            //     tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])
 
-                        // archive all artifacts
+                            // release the port reservation
-                        archiveArtifacts allowEmptyArchive: true,
+                            if (os == 'linux' || os == 'mac') {
-                            artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
+                                sh "Installation/Pipeline/port.sh --clean ${port}"
-                            defaultExcludes: false
+                            }
+                            else if (os == 'windows') {
+                                powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
+                            }
+
+                            // archive all artifacts
+                            archiveArtifacts allowEmptyArchive: true,
+                                artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
+                                defaultExcludes: false
+                        }
                     }
                 }
             }
@@ -741,11 +742,8 @@ def testStepParallel(os, edition, modeList) {
 
     for (mode in modeList) {
         for (engine in ['mmfiles', 'rocksdb']) {
-            if (testCheck(os, edition, mode, engine)) {
+            def name = "test-${os}-${edition}-${mode}-${engine}";
-                def name = "test-${os}-${edition}-${mode}-${engine}";
+            branches[name] = testStep(os, edition, mode, engine, name)
-
-                branches[name] = testStep(os, edition, mode, engine, name)
-            }
         }
     }
 
@@ -937,35 +935,36 @@ def buildStepCheck(os, edition) {
 
 def runEdition(os, edition) {
     return {
-        node(buildJenkins[os]) {
+        if (buildStepCheck(os, edition)) {
-            stage("build-${os}-${edition}") {
+            node(buildJenkins[os]) {
-                timeout(30) {
+                stage("build-${os}-${edition}") {
-                    checkoutCommunity()
+                    timeout(30) {
-                    checkCommitMessages()
+                        checkoutCommunity()
 
-                    if (edition == "enterprise") {
+                        if (edition == "enterprise") {
-                        checkoutEnterprise()
+                            checkoutEnterprise()
+                        }
+
+                        // checkoutResilience()
                     }
 
-                    // checkoutResilience()
+                    timeout(90) {
+                        buildEdition(os, edition)
+                        stashBinaries(os, edition)
+                    }
                 }
 
-                timeout(90) {
+                // we only need one jslint test per edition
-                    buildEdition(os, edition)
+                if (os == "linux") {
-                    stashBinaries(os, edition)
+                    stage("jslint-${edition}") {
+                        echo "Running jslint for ${edition}"
+                        jslint()
+                    }
                 }
             }
 
-            // we only need one jslint test per edition
+            testStepParallel(os, edition, ['cluster', 'singleserver'])
-            if (os == "linux") {
-                stage("jslint-${edition}") {
-                    echo "Running jslint for ${edition}"
-                    jslint()
-                }
-            }
         }
-
-        testStepParallel(os, edition, ['cluster', 'singleserver'])
     }
 }
 
@@ -978,13 +977,12 @@ def runOperatingSystems(osList) {
 
     for (os in osList) {
         for (edition in ['community', 'enterprise']) {
-            if (buildStepCheck(os, edition)) {
+            branches["build-${os}-${edition}"] = runEdition(os, edition)
-                branches["build-${os}-${edition}"] = runEdition(os, edition)
-            }
         }
     }
 
     parallel branches
 }
 
+checkCommitMessages()
 runOperatingSystems(['linux', 'mac', 'windows'])