Feature/jenkins pipeline (#3241)

Installation/Pipeline/Jenkinsfile.groovy

@@ -22,6 +22,8 @@ def defaultBuild = true
 def defaultCleanBuild = false
 def defaultCommunity = true
 def defaultEnterprise = true
+def defaultMaintainer = true
+def defaultUser = false
 // def defaultRunResilience = false
 def defaultRunTests = true
 
@@ -57,6 +59,16 @@ properties([
             description: 'build and run tests for enterprise',
             name: 'Enterprise'
         ),
+        booleanParam(
+            defaultValue: defaultMaintainer,
+            description: 'build in maintainer mode',
+            name: 'Maintainer'
+        ),
+        booleanParam(
+            defaultValue: defaultUser,
+            description: 'build in user (aka non-maintainer) mode',
+            name: 'User'
+        ),
         // booleanParam(
         //     defaultValue: defaultRunResilience,
         //     description: 'run resilience tests',
@@ -73,12 +85,6 @@ properties([
 // start with empty build directory
 cleanBuild = params.cleanBuild
 
-// build community
-useCommunity = params.Community
-
-// build enterprise
-useEnterprise = params.Enterprise
-
 // build linux
 useLinux = params.Linux
 
@@ -88,6 +94,18 @@ useMac = params.Mac
 // build windows
 useWindows = params.Windows
 
+// build and test community
+useCommunity = params.Community
+
+// build and test enterprise
+useEnterprise = params.Enterprise
+
+// build maintainer mode
+useMaintainer = params.Maintainer
+
+// build user mode
+useUser = params.User
+
 // run resilience tests
 //runResilience = params.runResilience
 
@@ -97,6 +115,9 @@ runTests = params.runTests
 // restrict builds
 restrictions = [:]
 
+// overview of configured builds and tests
+overview = ""
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                             CONSTANTS AND HELPERS
 // -----------------------------------------------------------------------------
@@ -150,6 +171,12 @@ def copyFile(os, src, dst) {
     }
 }
 
+def renameFolder(src, dst) {
+    fileOperations([
+        folderRenameOperation(destination: dst, source: src)
+    ])
+}
+
 def checkEnabledOS(os, text) {
     if (os == 'linux' && ! useLinux) {
         echo "Not ${text} ${os} because ${os} is not enabled"
@@ -183,35 +210,54 @@ def checkEnabledEdition(edition, text) {
     return true
 }
 
-def checkCoresAndSave(os, runDir, name, archRuns, archCores) {
+def checkEnabledMaintainer(maintainer, os, text) {
+    if (os == 'windows' && maintainer != 'maintainer') {
+        echo "Not ${text} ${maintainer} because ${maintainer} is not enabled under Windows"
+        return false
+    }
+
+    if (maintainer == 'maintainer' && ! useMaintainer) {
+        echo "Not ${text} ${maintainer} because ${maintainer} is not enabled"
+        return false
+    }
+
+    if (maintainer == 'user' && ! useUser) {
+        echo "Not ${text} ${maintainer} because ${maintainer} is not enabled"
+        return false
+    }
+
+    return true
+}
+
+def checkCoresAndSave(os, runDir, name, archRun) {
     if (os == 'windows') {
-        powershell "move-item -Force -ErrorAction Ignore ${runDir}/logs ${archRuns}/${name}.logs"
+        powershell "move-item -Force -ErrorAction Ignore ${runDir}/logs ${archRun}/${name}.logs"
-        powershell "move-item -Force -ErrorAction Ignore ${runDir}/out ${archRuns}/${name}.logs"
+        powershell "move-item -Force -ErrorAction Ignore ${runDir}/out ${archRun}/${name}.logs"
-        powershell "move-item -Force -ErrorAction Ignore ${runDir}/tmp ${archRuns}/${name}.tmp"
+        powershell "move-item -Force -ErrorAction Ignore ${runDir}/tmp ${archRun}/${name}.tmp"
 
         def files = findFiles(glob: "${runDir}/*.dmp")
         
         if (files.length > 0) {
             for (file in files) {
-                powershell "move-item -Force -ErrorAction Ignore ${file} ${archCores}"
+                powershell "move-item -Force -ErrorAction Ignore ${file} ${archRun}"
             }
 
-            powershell "copy-item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${archCores}"
+            powershell "copy-item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${archRun}"
 
             error("found dmp file")
         }
     }
     else {
-        sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"
+        sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRun}/${name}.\$i\" || true; done"
 
         def files = findFiles(glob: '${runDir}/core*')
 
         if (files.length > 0) {
             for (file in files) {
-                sh "mv ${file} ${archCores}"
+                sh "mv ${file} ${archRun}"
             }
 
-            sh "cp -a build/bin/* ${archCores}"
+            sh "cp -a build/bin/* ${archRun}"
 
             error("found core file")
         }
@@ -361,9 +407,14 @@ def checkCommitMessages() {
     else if (skip) {
         useLinux = false
         useMac = false
+
         useWindows = false
         useCommunity = false
         useEnterprise = false
+
+        useMaintainer = false
+        useUser = false
+
         // runResilience = false
         runTests = false
     }
@@ -375,56 +426,79 @@ def checkCommitMessages() {
             echo "build of PR"
 
             restrictions = [
-                "build-community-linux" : true,
+                // OS EDITION MAINTAINER
-                "build-community-mac" : true,
+                "build-linux-community-maintainer" : true,
-                "build-community-windows" : true,
+                "build-linux-enterprise-maintainer" : true,
-                "build-enterprise-linux" : true,
+                "build-mac-community-maintainer" : true,
-                "build-enterprise-mac" : true,
+                "build-mac-enterprise-user" : true,
-                "build-enterprise-windows" : true,
+                "build-windows-community-maintainer" : true,
-                "test-cluster-community-mmfiles-linux" : true,
+                "build-windows-enterprise-maintainer" : true,
-                "test-cluster-community-rocksdb-linux" : true,
+
-                "test-cluster-enterprise-mmfiles-linux" : true,
+                // OS EDITION MAINTAINER MODE ENGINE
-                "test-cluster-enterprise-rocksdb-linux" : true,
+                "test-linux-enterprise-maintainer-cluster-rocksdb" : true,
-                "test-singleserver-community-mmfiles-linux" : true,
+                "test-linux-community-maintainer-singleserver-mmfiles" : true
-                "test-singleserver-community-rocksdb-linux" : true,
-                "test-singleserver-enterprise-mmfiles-linux" : true,
-                "test-singleserver-enterprise-rocksdb-linux" : true
             ]
         }
         else {
             restrictions = [
-                "build-community-mac" : true,
+                // OS EDITION MAINTAINER
-                "build-enterprise-linux" : true,
+                "build-linux-community-user" : true,
-                "test-cluster-enterprise-rocksdb-linux" : true,
+                "build-linux-enterprise-maintainer" : true,
-                "test-singleserver-enterprise-mmfiles-linux" : true
+
+                // OS EDITION MAINTAINER MODE ENGINE
+                "test-linux-enterprise-maintainer-cluster-rocksdb" : true,
+                "test-linux-community-user-singleserver-mmfiles" : true
             ]
         }
     }
 
-    echo """BRANCH_NAME: ${env.BRANCH_NAME}
+    overview = """BRANCH_NAME: ${env.BRANCH_NAME}
 SOURCE: ${sourceBranchLabel}
 CHANGE_ID: ${env.CHANGE_ID}
 CHANGE_TARGET: ${env.CHANGE_TARGET}
 JOB_NAME: ${env.JOB_NAME}
 CAUSE: ${causeDescription}
+"""
 
-Linux: ${useLinux}
+    if (restrictions) {
+        useLinux = true
+        useMac = true
+        useWindows = true
+
+        useCommunity = true
+        useEnterprise = true
+
+        useMaintainer = true
+        useUser = true
+
+        // runResilience = true
+        runTests = true
+
+        overview += "Restrictions:\n"
+
+        for (r in restrictions.keySet()) {
+            overview += "    " + r + "\n"
+        }
+    }
+    else {
+        overview += """Linux: ${useLinux}
 Mac: ${useMac}
 Windows: ${useWindows}
 Clean Build: ${cleanBuild}
 Building Community: ${useCommunity}
 Building Enterprise: ${useEnterprise}
+Building Maintainer: ${useMaintainer}
+Building Non-Maintainer: ${useUser}
 Running Tests: ${runTests}
-
-Restrictions: ${restrictions.keySet().join(", ")}
 """
+    }
 }
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                     SCRIPTS STASH
 // -----------------------------------------------------------------------------
 
-def stashBinaries(os, edition) {
+def stashBinaries(os, edition, maintainer) {
     def paths = ["build/etc", "etc", "Installation/Pipeline", "js", "scripts", "UnitTests"]
 
     if (edition == "enterprise") {
@@ -445,25 +519,25 @@ def stashBinaries(os, edition) {
         // this is a super mega mess...scp will run as the system user and not as jenkins when run as a server
         // I couldn't figure out how to properly get it running for hours...so last resort was to install putty
 
-        powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk stash.zip jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.zip"
+        powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk stash.zip jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.zip"
     }
     else {
         paths << "build/bin/"
         paths << "build/tests/"
 
         sh "GZIP=-1 tar cpzf stash.tar.gz " + paths.join(" ")
-        sh "scp stash.tar.gz c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.tar.gz"
+        sh "scp stash.tar.gz c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.tar.gz"
     }
 }
 
-def unstashBinaries(os, edition) {
+def unstashBinaries(os, edition, maintainer) {
     if (os == "windows") {
-        powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.zip stash.zip"
+        powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.zip stash.zip"
         powershell "Expand-Archive -Path stash.zip -Force -DestinationPath ."
         powershell "copy build\\bin\\RelWithDebInfo\\* build\\bin"
     }
     else {
-        sh "scp c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.tar.gz stash.tar.gz"
+        sh "scp c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.tar.gz stash.tar.gz"
         sh "tar xpzf stash.tar.gz"
     }
 }
@@ -472,15 +546,40 @@ def unstashBinaries(os, edition) {
 // --SECTION--                                                    SCRIPTS JSLINT
 // -----------------------------------------------------------------------------
 
-def jslint() {
+def jslint(os, edition, maintainer) {
-    sh './Installation/Pipeline/test_jslint.sh'
+    def archDir  = "${os}-${edition}-${maintainer}"
+    def arch     = "${archDir}/02-jslint"
+    def archFail = "${archDir}/02-jslint-FAIL"
+
+    fileOperations([
+        folderDeleteOperation(arch),
+        folderDeleteOperation(archFail),
+        folderCreateOperation(arch)
+    ])
+
+    def logFile = "${arch}/jslint.log"
+
+    try {
+        sh "./Installation/Pipeline/test_jslint.sh | tee ${logFile}"
+        sh "if grep ERROR ${logFile}; then exit 1; fi"
+    }
+    catch (exc) {
+        renameFolder(arch, archFail)
+        fileOperations([fileCreateOperation(fileContent: 'BUILD FAILED', fileName: "${arch}-FAIL.txt")])
+        throw exc
+    }
+    finally {
+        archiveArtifacts allowEmptyArchive: true,
+            artifacts: "${arch}-*, ${arch}/**, ${archFail}/**",
+            defaultExcludes: false
+    }
 }
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                     SCRIPTS TESTS
 // -----------------------------------------------------------------------------
 
-def getTests(os, edition, mode, engine) {
+def getTests(os, edition, maintainer, mode, engine) {
     def tests = [
         ["arangobench", "arangobench" , ""],
         ["arangosh", "arangosh", "--skipShebang true"],
@@ -525,7 +624,7 @@ def getTests(os, edition, mode, engine) {
     return tests
 }
 
-def setupTestEnvironment(os, edition, logFile, runDir) {
+def setupTestEnvironment(os, edition, maintainer, logFile, runDir) {
     fileOperations([
         folderCreateOperation("${runDir}/tmp"),
     ])
@@ -550,10 +649,10 @@ def setupTestEnvironment(os, edition, logFile, runDir) {
     }
 }
 
-def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed, archCores) {
+def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch, stageName) {
     def parallelity = 4
     def testIndex = 0
-    def tests = getTests(os, edition, mode, engine)
+    def tests = getTests(os, edition, maintainer, mode, engine)
 
     def portInterval = (mode == "cluster") ? 40 : 10
 
@@ -578,11 +677,13 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
             testArgs += " --cluster true"
         }
 
-        testMap["test-${os}-${edition}-${mode}-${engine}-${name}"] = {
+        testMap["${stageName}-${name}"] = {
-            def logFile = "${arch}/${name}.log"
+            def logFile       = pwd() + "/" + "${arch}/${name}.log"
-            def logFileFailed = "${archFailed}/${name}.log"
+            def logFileFailed = pwd() + "/" + "${arch}-FAIL/${name}.log"
+            def archRun       = pwd() + "/" + "${arch}-RUN"
+
             def runDir = "run.${currentIndex}"
-            def port = portInit + currentIndex * portInterval
+            def port   = portInit + currentIndex * portInterval
 
             testArgs += " --minPort " + port
             testArgs += " --maxPort " + (port + portInterval - 1)
@@ -594,10 +695,15 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
                           testArgs
 
             try {
-                lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${engine}-${lockIndex}") {
+                lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${maintainer}-${engine}-${lockIndex}") {
-                    setupTestEnvironment(os, edition, logFile, runDir)
+                    setupTestEnvironment(os, edition, maintainer, logFile, runDir)
 
                     try {
+                        // seriously...30 minutes is the super absolute max max max.
+                        // even in the worst situations ArangoDB MUST be able to finish within 60 minutes
+                        // even if the features are green this is completely broken performance wise..
+                        // DO NOT INCREASE!!
+
                         timeout(30) {
                             def tmpDir = pwd() + "/" + runDir + "/tmp"
 
@@ -622,11 +728,23 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
                     catch (exc) {
                         echo "caught error, copying log to ${logFileFailed}"
                         echo exc.toString()
+
+                        fileOperations([
+                            fileCreateOperation(fileContent: 'TEST FAILED', fileName: "${archDir}-FAIL.txt")
+                        ])
+
                         copyFile(os, logFile, logFileFailed)
                         throw exc
                     }
                     finally {
-                        checkCoresAndSave(os, runDir, name, archRuns, archCores)
+                        def logFileRel       = "${arch}/${name}.log"
+                        def logFileFailedRel = "${arch}-FAIL/${name}.log"
+
+                        checkCoresAndSave(os, runDir, name, archRun)
+
+                        archiveArtifacts allowEmptyArchive: true,
+                            artifacts: "${logFileRel}, ${logFileFailedRel}",
+                            defaultExcludes: false
                     }
                 }
             }
@@ -641,7 +759,12 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
     parallel testSteps
 }
 
-def testCheck(os, edition, mode, engine) {
+def testCheck(os, edition, maintainer, mode, engine) {
+    if (! runTests) {
+        echo "Not testing ${os} ${mode} because testing is not enabled"
+        return false
+    }
+
     if (! checkEnabledOS(os, 'testing')) {
        return false
     }
@@ -650,32 +773,26 @@ def testCheck(os, edition, mode, engine) {
        return false
     }
 
-    if (! runTests) {
+    if (! checkEnabledMaintainer(maintainer, os, 'building')) {
-        echo "Not testing ${os} ${mode} because testing is not enabled"
+       return false
-        return false
     }
 
-    if (restrictions && !restrictions["test-${mode}-${edition}-${engine}-${os}"]) {
+    if (restrictions && !restrictions["test-${os}-${edition}-${maintainer}-${mode}-${engine}"]) {
         return false
     }
 
     return true
 }
 
-def testStep(os, edition, mode, engine, testName) {
+def testStep(os, edition, maintainer, mode, engine, stageName) {
     return {
-        if (testCheck(os, edition, mode, engine)) {
+        if (testCheck(os, edition, maintainer, mode, engine)) {
             node(testJenkins[os]) {
-                stage(testName) {
+                stage(stageName) {
-                    def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
+                    def archDir    = "${os}-${edition}-${maintainer}"
-                    def archFailedRel = "${archRel}_FAILED"
+                    def arch       = "${archDir}/03-test-${mode}-${engine}"
-                    def archRunsRel = "${archRel}_RUN"
+                    def archFail   = "${arch}-FAIL"
-                    def archCoresRel = "${archRel}_CORES"
+                    def archRun    = "${arch}-RUN"
-
-                    def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
-                    def archFailed = "${arch}_FAILED"
-                    def archRuns = "${arch}_RUN"
-                    def archCores = "${arch}_CORES"
 
                     // clean the current workspace completely
                     deleteDir()
@@ -683,45 +800,33 @@ def testStep(os, edition, mode, engine, testName) {
                     // create directories for the artifacts
                     fileOperations([
                         folderCreateOperation(arch),
-                        folderCreateOperation(archFailed),
+                        folderCreateOperation(archFail),
-                        folderCreateOperation(archRuns),
+                        folderCreateOperation(archRun)
-                        folderCreateOperation(archCores)
                     ])
 
                     // unstash binaries
-                    unstashBinaries(os, edition)
+                    unstashBinaries(os, edition, maintainer)
 
                     // find a suitable port
                     def port = (getStartPort(os) as Integer)
                     echo "Using start port: ${port}"
 
-                    // seriously...60 minutes is the super absolute max max max.
+                    try {
-                    // even in the worst situations ArangoDB MUST be able to finish within 60 minutes
+                        executeTests(os, edition, maintainer, mode, engine, port, archDir, arch, stageName)
-                    // even if the features are green this is completely broken performance wise..
+                    }
-                    // DO NOT INCREASE!!
+                    finally {
-
+                        // release the port reservation
-                    timeout(60) {
+                        if (os == 'linux' || os == 'mac') {
-                        try {
+                            sh "Installation/Pipeline/port.sh --clean ${port}"
-                            executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
                         }
-                        finally {
+                        else if (os == 'windows') {
-                            // step([$class: 'XUnitBuilder',
+                            powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
-                            //     thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
-                            //     tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])
-
-                            // release the port reservation
-                            if (os == 'linux' || os == 'mac') {
-                                sh "Installation/Pipeline/port.sh --clean ${port}"
-                            }
-                            else if (os == 'windows') {
-                                powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
-                            }
-
-                            // archive all artifacts
-                            archiveArtifacts allowEmptyArchive: true,
-                                artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
-                                defaultExcludes: false
                         }
+
+                        // archive all artifacts
+                        archiveArtifacts allowEmptyArchive: true,
+                            artifacts: "${arch}-*, ${archRun}/**",
+                            defaultExcludes: false
                     }
                 }
             }
@@ -729,13 +834,13 @@ def testStep(os, edition, mode, engine, testName) {
     }
 }
 
-def testStepParallel(os, edition, modeList) {
+def testStepParallel(os, edition, maintainer, modeList) {
     def branches = [:]
 
     for (mode in modeList) {
         for (engine in ['mmfiles', 'rocksdb']) {
-            def name = "test-${os}-${edition}-${mode}-${engine}";
+            def stageName = "test-${os}-${edition}-${maintainer}-${mode}-${engine}";
-            branches[name] = testStep(os, edition, mode, engine, name)
+            branches[stageName] = testStep(os, edition, maintainer, mode, engine, stageName)
         }
     }
 
@@ -870,66 +975,90 @@ def testStepParallel(os, edition, modeList) {
 // --SECTION--                                                     SCRIPTS BUILD
 // -----------------------------------------------------------------------------
 
-def buildEdition(os, edition) {
+def buildEdition(os, edition, maintainer) {
-    def arch = "01_build_${os}_${edition}"
+    def archDir  = "${os}-${edition}-${maintainer}"
+    def arch     = "${archDir}/01-build"
+    def archFail = "${archDir}/01-build-FAIL"
 
     fileOperations([
         folderDeleteOperation(arch),
+        fileDeleteOperation(excludes: '', includes: "${archDir}-*"),
+        folderDeleteOperation(archFail),
         folderCreateOperation(arch)
     ])
 
+    def logFile = "${arch}/build.log"
+
     try {
-        if (os == 'linux') {
+        if (os == 'linux' || os == 'mac') {
-            sh "./Installation/Pipeline/linux/build_${os}_${edition}.sh 64 ${arch}"
+            sh "echo \"Host: `hostname`\" | tee ${logFile}"
-        }
+            sh "echo \"PWD:  `pwd`\" | tee -a ${logFile}"
-        else if (os == 'mac') {
+            sh "echo \"Date: `date`\" | tee -a ${logFile}"
-            sh "./Installation/Pipeline/mac/build_${os}_${edition}.sh 16 ${arch}"
-        }
-        else if (os == 'windows') {
-            // I concede...we need a lock for windows...I could not get it to run concurrently...
-            // v8 would not build multiple times at the same time on the same machine:
-            // PDB API call failed, error code '24': ' etc etc
-            // in theory it should be possible to parallelize it by setting an environment variable
-            // (see the build script) but for v8 it won't work :(
-            // feel free to recheck if there is time somewhen...this thing here really should not be possible but
-            // ensure that there are 2 concurrent builds on the SAME node building v8 at the same time to properly
-            // test it. I just don't want any more "yeah that might randomly fail. just restart" sentences any more.
 
-            def hostname = powershell(returnStdout: true, script: "hostname")
+            if (os == 'linux') {
-
+                sh "./Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh 64 ${os} ${edition} ${maintainer} ${arch}"
-            lock('build-${hostname}') {
+            }
-                powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}.ps1"
+            else if (os == 'mac') {
+                sh "./Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh 16 ${os} ${edition} ${maintainer} ${arch}"
             }
         }
+        else if (os == 'windows') {
+            def tmpDir = "${arch}/tmp"
+
+            fileOperations([
+                folderCreateOperation(tmpDir)
+            ])
+
+            // withEnv(["TMPDIR=${tmpDir}", "TEMPDIR=${tmpDir}", "TMP=${tmpDir}",
+            //          "_MSPDBSRV_ENDPOINT_=${edition}-${env.BUILD_TAG}", "GYP_USE_SEPARATE_MSPDBSRV=1"]) {
+                powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}.ps1"
+            // }
+
+            fileOperations([
+                folderDeleteOperation(tmpDir)
+            ])
+        }
+    }
+    catch (exc) {
+        fileOperations([
+            fileCreateOperation(fileContent: 'BUILD FAILED', fileName: "${archDir}-FAIL.txt")
+        ])
+
+        renameFolder(arch, archFail)
+        throw exc
     }
     finally {
         archiveArtifacts allowEmptyArchive: true,
-            artifacts: "${arch}/**",
+            artifacts: "${archDir}-*, ${arch}/**, ${archFail}/**",
             defaultExcludes: false
     }
 }
 
-def buildStepCheck(os, edition) {
+def buildStepCheck(os, edition, maintainer) {
     if (! checkEnabledOS(os, 'building')) {
        return false
     }
 
-    if (! checkEnabledEdition(edition, 'testing')) {
+    if (! checkEnabledEdition(edition, 'building')) {
        return false
     }
 
-    if (restrictions && !restrictions["build-${edition}-${os}"]) {
+    if (! checkEnabledMaintainer(maintainer, os, 'building')) {
+       return false
+    }
+
+    if (restrictions && !restrictions["build-${os}-${edition}-${maintainer}"]) {
         return false
     }
 
     return true
 }
 
-def runEdition(os, edition) {
+def runEdition(os, edition, maintainer, stageName) {
     return {
-        if (buildStepCheck(os, edition)) {
+        if (buildStepCheck(os, edition, maintainer)) {
             node(buildJenkins[os]) {
-                stage("build-${os}-${edition}") {
+                stage(stageName) {
                     timeout(30) {
                         checkoutCommunity()
 
@@ -940,9 +1069,30 @@ def runEdition(os, edition) {
                         // checkoutResilience()
                     }
 
-                    timeout(90) {
+                    // I concede...we need a lock for windows...I could not get it to run concurrently...
-                        buildEdition(os, edition)
+                    // v8 would not build multiple times at the same time on the same machine:
-                        stashBinaries(os, edition)
+                    // PDB API call failed, error code '24': ' etc etc
+                    // in theory it should be possible to parallelize it by setting an environment variable
+                    // (see the build script) but for v8 it won't work :(
+                    // feel free to recheck if there is time somewhen...this thing here really should not be possible but
+                    // ensure that there are 2 concurrent builds on the SAME node building v8 at the same time to properly
+                    // test it. I just don't want any more "yeah that might randomly fail. just restart" sentences any more.
+
+                    if (os == "windows") {
+                        def hostname = powershell(returnStdout: true, script: "hostname").trim()
+
+                        lock("build-windows-${hostname}") {
+                            timeout(90) {
+                                buildEdition(os, edition, maintainer)
+                                stashBinaries(os, edition, maintainer)
+                            }
+                        }
+                    }
+                    else {
+                        timeout(90) {
+                            buildEdition(os, edition, maintainer)
+                            stashBinaries(os, edition, maintainer)
+                        }
                     }
                 }
 
@@ -950,12 +1100,12 @@ def runEdition(os, edition) {
                 if (os == "linux") {
                     stage("jslint-${edition}") {
                         echo "Running jslint for ${edition}"
-                        jslint()
+                        jslint(os, edition, maintainer)
                     }
                 }
             }
 
-            testStepParallel(os, edition, ['cluster', 'singleserver'])
+            testStepParallel(os, edition, maintainer, ['cluster', 'singleserver'])
         }
     }
 }
@@ -969,7 +1119,10 @@ def runOperatingSystems(osList) {
 
     for (os in osList) {
         for (edition in ['community', 'enterprise']) {
-            branches["build-${os}-${edition}"] = runEdition(os, edition)
+            for (maintainer in ['maintainer', 'user']) {
+                def stageName = "build-${os}-${edition}-${maintainer}"
+                branches[stageName] = runEdition(os, edition, maintainer, stageName)
+            }
         }
     }
 
@@ -978,5 +1131,11 @@ def runOperatingSystems(osList) {
 
 timestamps {
     checkCommitMessages()
+
+    node("master") {
+        fileOperations([fileCreateOperation(fileContent: overview, fileName: "overview.txt")])
+        archiveArtifacts(allowEmptyArchive: true, artifacts: "overview.txt")
+    }
+
     runOperatingSystems(['linux', 'mac', 'windows'])
-}
+}

Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh

@@ -1,7 +1,9 @@
+#!/bin/bash
 concurrency=$1
-edition=$2
+os=$2
-os=$3
+edition=$3
-logdir=$4
+maintainer=$4
+logdir=$5
 
 ENTERPRISE=""
 type="build"
@@ -17,6 +19,19 @@ else
     exit 1
 fi
 
+MAINTAINER=""
+
+if [ "$maintainer" == maintainer ]; then
+    MAINTAINER="-DUSE_MAINTAINER_MODE=On"
+    type="${type}_maintainer"
+elif [ "$maintainer" == user ]; then
+    MAINTAINER="-DUSE_MAINTAINER_MODE=Off"
+    type="${type}_user"
+else
+    echo "$0: unknown maintainer '$maintainer', expecting 'maintainer' or 'user'"
+    exit 1
+fi
+
 if [ "$os" == linux ]; then
     type="${type}_linux"
     load=40
@@ -28,10 +43,6 @@ else
     exit 1
 fi
 
-echo "CONCURRENY: $concurrency"
-echo "HOST: `hostname`"
-echo "PWD: `pwd`"
-
 mkdir -p build
 
 if [ -z "$logdir" ]; then
@@ -42,6 +53,10 @@ fi
 
 touch $logdir/build.log
 
+echo "CONCURRENY: $concurrency" | tee -a $logdir/build.log
+echo "HOST: `hostname`" | tee -a $logdir/build.log
+echo "PWD: `pwd`" | tee -a $logdir/build.log
+
 (
     set -eo pipefail
     cd build
@@ -50,15 +65,15 @@ touch $logdir/build.log
     CXXFLAGS=-fno-omit-frame-pointer \
         cmake \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-            -DUSE_MAINTAINER_MODE=On \
             -DUSE_CATCH_TESTS=On \
             -DUSE_FAILURE_TESTS=On \
             -DDEBUG_SYNC_REPLICATION=On \
+            $MAINTAINER \
             $ENTERPRISE \
-            ..  2>&1 | tee ../$logdir/build.log
+            ..  2>&1 | tee -a ../$logdir/build.log
 
     echo "`date +%T` building..."
     make -j $concurrency -l $load 2>&1 | tee -a ../$logdir/build.log
 ) || exit 1
 
-echo "`date +%T` done..."
+echo "`date +%T` done..."

Installation/Pipeline/linux/build_linux_community.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" community linux "$2"

Installation/Pipeline/linux/build_linux_enterprise.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" enterprise linux "$2"

Installation/Pipeline/mac/build_mac_community.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" community mac "$2"

Installation/Pipeline/mac/build_mac_enterprise.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" enterprise mac "$2"

Installation/Pipeline/windows/build_windows_community.ps1

@@ -1,10 +1,12 @@
-$logdir="01_build_windows_community"
+$logdir="windows-community-maintainer/01-build"
+
 $ErrorActionPreference="Stop"
 $vcpath=$(Get-ItemProperty HKLM:\SOFTWARE\Wow6432Node\Microsoft\VisualStudio\SxS\VC7)."14.0"
-#$env:_MSPDBSRV_ENDPOINT_="community-${env:BUILD_TAG}"
 $buildOptions = "-DUSE_MAINTAINER_MODE=On -DUSE_ENTERPRISE=Off -DUSE_CATCH_TESTS=On -DUSE_FAILURE_TESTS=On -DDEBUG_SYNC_REPLICATION=On -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSKIP_PACKAGING=On"
+
 Remove-Item -Force -Recurse ${logdir} -ErrorAction SilentlyContinue
 New-Item -Force -ItemType Directory ${logdir} -ErrorAction SilentlyContinue
+
 if (Get-Command docker -errorAction SilentlyContinue) {
   $buildOptions += " -DOPENSSL_INCLUDE_DIR=`"`$env:OPENSSL_INCLUDE_DIR`" -DLIB_EAY_RELEASE=`"`$env:LIB_EAY_RELEASE`" -DSSL_EAY_RELEASE=`"`$env:SSL_EAY_RELEASE`" -DLIB_EAY_RELEASE_DLL=`"`$env:LIB_EAY_RELEASE_DLL`" -DSSL_EAY_RELEASE_DLL=`"`$env:SSL_EAY_RELEASE_DLL"
   $volume = "$env:WORKSPACE"
@@ -22,9 +24,16 @@ exit $LastExitCode
   docker run --rm -v $volume m0ppers/build-container powershell C:\arangodb\buildscript.ps1 | Set-Content -PassThru ${logdir}\build.log
 } else {
   $env:GYP_MSVS_OVERRIDE_PATH="${vcpath}\bin"
+  $env:CC="${vcpath}\bin\cl.exe"
+  $env:CXX="${vcpath}\bin\cl.exe"
+
+  $env
+
   New-Item -ItemType Directory -Force -Path build
   cd build
+
   Invoke-Expression "cmake .. -G `"Visual Studio 15 2017 Win64`" ${buildOptions} | Set-Content -PassThru ..\${logdir}\build.log"
   cmake --build . --config RelWithDebInfo | Add-Content -PassThru ..\${logdir}\build.log
+
   cd ..
 }

Installation/Pipeline/windows/build_windows_enterprise.ps1

@@ -1,10 +1,12 @@
-$logdir="01_build_windows_enterprise"
+$logdir="windows-enterprise-maintainer/01-build"
+
 $ErrorActionPreference="Stop"
 $vcpath=$(Get-ItemProperty HKLM:\SOFTWARE\Wow6432Node\Microsoft\VisualStudio\SxS\VC7)."14.0"
-#$env:_MSPDBSRV_ENDPOINT_="enterprise-${env:BUILD_TAG}"
 $buildOptions = "-DUSE_MAINTAINER_MODE=On -DUSE_ENTERPRISE=On -DUSE_CATCH_TESTS=On -DUSE_FAILURE_TESTS=On -DDEBUG_SYNC_REPLICATION=On -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSKIP_PACKAGING=On"
+
 Remove-Item -Force -Recurse ${logdir} -ErrorAction SilentlyContinue
 New-Item -Force -ItemType Directory ${logdir} -ErrorAction SilentlyContinue
+
 if (Get-Command docker -errorAction SilentlyContinue) {
   $buildOptions += " -DOPENSSL_INCLUDE_DIR=`"`$env:OPENSSL_INCLUDE_DIR`" -DLIB_EAY_RELEASE=`"`$env:LIB_EAY_RELEASE`" -DSSL_EAY_RELEASE=`"`$env:SSL_EAY_RELEASE`" -DLIB_EAY_RELEASE_DLL=`"`$env:LIB_EAY_RELEASE_DLL`" -DSSL_EAY_RELEASE_DLL=`"`$env:SSL_EAY_RELEASE_DLL"
   $volume = "$env:WORKSPACE"
@@ -22,9 +24,16 @@ exit $LastExitCode
   docker run --rm -v $volume m0ppers/build-container powershell C:\arangodb\buildscript.ps1 | Set-Content -PassThru ${logdir}\build.log
 } else {
   $env:GYP_MSVS_OVERRIDE_PATH="${vcpath}\bin"
+  $env:CC="${vcpath}\bin\cl.exe"
+  $env:CXX="${vcpath}\bin\cl.exe"
+
+  $env
+
   New-Item -ItemType Directory -Force -Path build
   cd build
+
   Invoke-Expression "cmake .. -G `"Visual Studio 15 2017 Win64`" ${buildOptions} | Set-Content -PassThru ..\${logdir}\build.log"
   cmake --build . --config RelWithDebInfo | Add-Content -PassThru ..\${logdir}\build.log
+
   cd ..
 }