From 5105da9c339db7decccedc0dbe2f1cbef2abd11f Mon Sep 17 00:00:00 2001 From: Frank Celler Date: Thu, 22 Jun 2017 22:58:53 +0200 Subject: [PATCH] Feature/jenkins pipeline (#2653) --- Installation/Pipeline/Jenkinsfile.feature | 495 ++++++++++-------- .../Pipeline/include/build_EDITION_OS.inc | 24 +- 2 files changed, 287 insertions(+), 232 deletions(-) diff --git a/Installation/Pipeline/Jenkinsfile.feature b/Installation/Pipeline/Jenkinsfile.feature index 440b11e12c..2baf28dc8b 100644 --- a/Installation/Pipeline/Jenkinsfile.feature +++ b/Installation/Pipeline/Jenkinsfile.feature @@ -4,36 +4,62 @@ properties([ parameters([ booleanParam( defaultValue: false, - description: 'clean build', + description: 'clean build directories', name: 'cleanBuild' ), booleanParam( defaultValue: true, - description: 'build enterprise', + description: 'build and run tests for community', + name: 'buildCommunity' + ), + booleanParam( + defaultValue: false, + description: 'build and run tests for enterprise', name: 'buildEnterprise' ), booleanParam( defaultValue: true, - description: 'build Linux', + description: 'build and run tests on Linux', name: 'buildLinux' ), booleanParam( defaultValue: false, - description: 'build Mac', + description: 'build and run tests on Mac', name: 'buildMac' ), booleanParam( defaultValue: false, - description: 'build Windows', + description: 'build and run tests in Windows', name: 'buildWindows' ), + booleanParam( + defaultValue: false, + description: 'run jslint', + name: 'runJslint' + ), + booleanParam( + defaultValue: false, + description: 'run resilience tests', + name: 'runResilience' + ), + booleanParam( + defaultValue: false, + description: 'run tests', + name: 'runTests' + ) ]) ]) // start with empty build directory cleanBuild = params.cleanBuild -// build enterprise version +// build all combinations +buildFull = false + +// build community +buildCommunity = params.buildCommunity + +// build enterprise buildEnterprise = params.buildEnterprise // build linux @@ -45,10 +71,22 @@ buildMac = params.buildMac // build windows buildWindows = params.buildWindows +// run jslint +runJslint = params.runJslint + +// run resilience tests +runResilience = params.runResilience + +// run tests +runTests = params.runTests + // ----------------------------------------------------------------------------- // --SECTION-- CONSTANTS AND HELPERS // ----------------------------------------------------------------------------- +// github repositiory for resilience tests +resilienceRepo = 'https://github.com/arangodb/resilience-tests' + // github repositiory for enterprise version enterpriseRepo = 'https://github.com/arangodb/enterprise' @@ -64,7 +102,7 @@ def PowerShell(psCmd) { } // ----------------------------------------------------------------------------- -// --SECTION-- SCRIPTS +// --SECTION-- SCRIPTS SCM // ----------------------------------------------------------------------------- def checkoutCommunity() { @@ -99,8 +137,6 @@ def checkoutEnterprise() { extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'enterprise']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: credentials, url: enterpriseRepo]]]) - - buildEnterprise = true } catch (exc) { echo "Failed ${env.BRANCH_NAME}, trying enterprise branch devel" @@ -120,6 +156,21 @@ def checkoutEnterprise() { sh 'cd enterprise && git clean -f -d -x' } +def checkoutResilience() { + checkout( + changelog: false, + poll: false, + scm: [ + $class: 'GitSCM', + branches: [[name: "*/master"]], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'resilience']], + submoduleCfg: [], + userRemoteConfigs: [[credentialsId: credentials, url: resilienceRepo]]]) + + sh 'cd resilience && git clean -f -d -x' +} + def checkCommitMessages() { def changeLogSets = currentBuild.changeSets @@ -146,46 +197,6 @@ def checkCommitMessages() { cleanBuild = false } - if (msg ==~ /(?i).*\[ci:[^\]]*linux[ \]].*/) { - echo "not building linux because message contained 'linux'" - buildLinux = true - } - - if (msg ==~ /(?i).*\[ci:[^\]]*no-linux[ \]].*/) { - echo "not building linux because message contained 'no-linux'" - buildLinux = false - } - - if (msg ==~ /(?i).*\[ci:[^\]]*mac[ \]].*/) { - echo "building mac because message contained 'mac'" - buildMac = true - } - - if (msg ==~ /(?i).*\[ci:[^\]]*no-mac[ \]].*/) { - echo "building mac because message contained 'no-mac'" - buildMac = false - } - - if (msg ==~ /(?i).*\[ci:[^\]]*windows[ \]].*/) { - echo "building windows because message contained 'windows'" - buildWindows = true - } - - if (msg ==~ /(?i).*\[ci:[^\]]*no-windows[ \]].*/) { - echo "building windows because message contained 'no-windows'" - buildWindows = false - } - - if (msg ==~ /(?i).*\[ci:[^\]]*enterprise[ \]].*/) { - echo "building enterprise because message contained 'enterprise'" - buildEnterprise = true - } - - if (msg ==~ /(?i).*\[ci:[^\]]*no-enterprise[ \]].*/) { - echo "building enterprise because message contained 'no-enterprise'" - buildEnterprise = false - } - def files = new ArrayList(entry.affectedFiles) for (int k = 0; k < files.size(); k++) { @@ -199,17 +210,25 @@ def checkCommitMessages() { } echo 'Clean Build: ' + (cleanBuild ? 'true' : 'false') + echo 'Build Community: ' + (buildCommunity ? 'true' : 'false') echo 'Build Enterprise: ' + (buildEnterprise ? 'true' : 'false') echo 'Build Linux: ' + (buildLinux ? 'true' : 'false') echo 'Build Mac: ' + (buildMac ? 'true' : 'false') echo 'Build Windows: ' + (buildWindows ? 'true' : 'false') + echo 'Run Jslint: ' + (runJslint ? 'true' : 'false') + echo 'Run Resilience: ' + (runResilience ? 'true' : 'false') + echo 'Run Tests: ' + (runTests ? 'true' : 'false') } +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS STASH +// ----------------------------------------------------------------------------- + def stashSourceCode() { sh 'rm -f source.*' if (buildLinux || buildMac) { - sh 'tar -c -z -f source.tar.gz --exclude "source.*" --exclude "*tmp" --exclude ".git" *' + sh 'tar -c -f source.tar --exclude "source.*" --exclude "*tmp" --exclude ".git" *' stash includes: 'source.*', name: 'sourceTar' } @@ -228,33 +247,52 @@ def unstashSourceCode(os) { if (os == 'linux' || os == 'mac') { unstash 'sourceTar' - sh 'tar -x -z -p -f source.tar.gz' + sh 'tar -x -p -f source.tar' sh 'mkdir -p artefacts' } else if (os == 'windows') { unstash 'source' + if (!fileExists('artefacts')) { bat 'mkdir artefacts' } } } +def stashBinaries(edition, os) { + if (edition == 'community') { + stash includes: binariesCommunity, name: 'build-' + edition + '-' + os + } + else if (edition == 'enterprise') { + stash includes: binariesEnterprise, name: 'build-' + edition + '-' + os + } +} + +def unstashBinaries(edition, os) { + sh 'rm -rf *' + unstash 'build-' + edition + '-' + os +} + +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS BUILD +// ----------------------------------------------------------------------------- + def buildEdition(edition, os) { try { if (os == 'linux' || os == 'mac') { - def tarfile = 'build-' + edition + '-' + os + '.tar.gz' + def tarfile = 'build-' + edition + '-' + os + '.tar' cache(maxCacheSize: 50000, caches: [ [$class: 'ArbitraryFileCache', includes: tarfile, path: 'artefacts']]) { if (!cleanBuild && fileExists('artefacts/' + tarfile)) { - sh 'tar -x -z -p -f artefacts/' + tarfile + sh 'tar -x -p -f artefacts/' + tarfile } sh 'rm -f artefacts/' + tarfile - sh './Installation/Pipeline/build_' + edition + '_' + os + '.sh 16' - sh 'tar -c -z -f artefacts/' + tarfile + ' build-' + edition + sh './Installation/Pipeline/build_' + edition + '_' + os + '.sh 64' + sh 'tar -c -f artefacts/' + tarfile + ' build-' + edition } } else if (os == 'windows') { @@ -274,18 +312,78 @@ def buildEdition(edition, os) { } } -def stashBinaries(edition, os) { - if (edition == 'community') { - stash includes: binariesCommunity, name: 'build-' + edition + '-' + os +def buildStep(os, edition, full) { + if (full && ! buildFull) { + echo "Not building combination " + os + " " + edition + " " + return } - else if (edition == 'enterprise') { - stash includes: binariesEnterprise, name: 'build-' + edition + '-' + os + + if (os == 'linux' && ! buildLinux) { + echo "Not building " + os + " version" + return + } + + if (os == 'mac' && ! buildMac) { + echo "Not building " + os + " version" + return + } + + if (os == 'windows' && ! buildWindows) { + echo "Not building " + os + " version" + return + } + + if (edition == 'enterprise' && ! buildEnterprise) { + echo "Not building " + edition + " version" + return + } + + if (edition == 'community' && ! buildCommunity) { + echo "Not building " + edition + " version" + return + } + + node(os) { + unstashSourceCode(os) + buildEdition(edition, os) + + if (runTests || runResilience) { + stashBinaries(edition, os) + } } } -def unstashBinaries(edition, os) { - sh 'rm -rf *' - unstash 'build-' + edition + '-' + os +// ----------------------------------------------------------------------------- +// --SECTION-- SCRIPTS TESTS +// ----------------------------------------------------------------------------- + +def jslint() { + try { + sh './Installation/Pipeline/test_jslint.sh' + } + catch (exc) { + echo exc.toString() + currentBuild.result = 'UNSTABLE' + } +} + +def jslintStep() { + if (runJslint) { + if (buildLinux) { + node(os) { + echo "Running jslint test" + + unstashBinaries('community', os) + jslint() + } + } + else { + echo "Not building " + os + " version" + } + } + else { + echo "Not running tests" + } } def testEdition(edition, os, type, engine) { @@ -301,13 +399,89 @@ def testEdition(edition, os, type, engine) { } } -def jslint() { - try { - sh './Installation/Pipeline/test_jslint.sh' +def testStep(edition, os, mode, engine, full) { + if (! runTests) { + echo "Not running tests" + return } - catch (exc) { - echo exc.toString() - currentBuild.result = 'UNSTABLE' + + if (full && ! buildFull) { + echo "Not building combination " + os + " " + edition + " " + return + } + + if (os == 'linux' && ! buildLinux) { + echo "Not building " + os + " version" + return + } + + if (os == 'mac' && ! buildMac) { + echo "Not building " + os + " version" + return + } + + if (os == 'windows' && ! buildWindows) { + echo "Not building " + os + " version" + return + } + + if (edition == 'enterprise' && ! buildEnterprise) { + echo "Not building " + edition + " version" + return + } + + if (edition == 'community' && ! buildCommunity) { + echo "Not building " + edition + " version" + return + } + + node(os) { + echo "Running " + mode + " " + edition + " " + engine + " " + os + " test" + + unstashBinaries(edition, os) + testEdition(edition, os, mode, engine) + } +} + +def testEditionResilience(edition, os, engine) { + echo "missing" + sh "ls -l" +} + +def testResilienceStep(os, edition, engine) { + if (! runResilience) { + echo "Not running resilience tests" + return + } + + if (os == 'linux' && ! buildLinux) { + echo "Not building " + os + " version" + return + } + + if (os == 'mac' && ! buildMac) { + echo "Not building " + os + " version" + return + } + + if (os == 'windows' && ! buildWindows) { + echo "Not building " + os + " version" + return + } + + if (edition == 'enterprise' && ! buildEnterprise) { + echo "Not building " + edition + " version" + return + } + + if (edition == 'community' && ! buildCommunity) { + echo "Not building " + edition + " version" + return + } + + node(os) { + unstashBinaries(edition, os) + testEditionResilience(edition, os, engine) } } @@ -320,6 +494,7 @@ stage('checkout') { checkoutCommunity() checkCommitMessages() checkoutEnterprise() + checkoutResilience() stashSourceCode() } } @@ -327,179 +502,39 @@ stage('checkout') { // cmake is very picky about the absolute path. Therefore never put a stage // into an `if` -stage('build linux') { - def os = 'linux' - +stage('build') { parallel( 'build-community-linux': { - def edition = 'community' + buildStep('linux', 'community', false) - if (buildLinux) { - node(os) { - unstashSourceCode(os) - buildEdition(edition, os) - stashBinaries(edition, os) - } - } - else { - echo "Not building " + os + " version" - } + parallel( + 'test-singleserver-community-mmfiles-linux': { testStep('community', 'linux', 'singleserver', 'mmfiles', true) }, + 'test-singleserver-community-rocksdb-linux': { testStep('community', 'linux', 'singleserver', 'rocksdb', false) }, + 'test-cluster-community-mmfiles-linux': { testStep('community', 'linux', 'cluster', 'mmfiles', false) }, + 'test-cluster-community-rocksdb-linux': { testStep('community', 'linux', 'cluster', 'rocksdb', true) } + ) }, - 'build-enterprise-linux': { - def edition = 'enterprise' + 'build-community-mac': { buildStep('mac', 'community', false) }, + 'build-community-windows': { buildStep('windows', 'community', false) }, - if (buildLinux) { - if (buildEnterprise) { - node(os) { - unstashSourceCode(os) - buildEdition(edition, os) - stashBinaries(edition, os) - } - } - else { - echo "Not building enterprise version" - } - } - else { - echo "Not building " + os + " version" - } - } + 'build-enterprise-linux': { buildStep('linux', 'enterprise', false) }, + 'build-enterprise-mac': { buildStep('mac', 'enterprise', true) }, + 'build-enterprise-windows': { buildStep('windows', 'enterprise', true) }, ) } -stage('test linux') { - def os = 'linux' - +stage('test') { parallel( - 'test-singleserver-community-rocksdb': { - def mode = 'singleserver' - def edition = 'community' - def engine = 'rocksdb' + 'test-cluster-enterprise-mmfiles-linux': { testStep('enterprise', 'linux', 'cluster', 'mmfiles', true) }, + 'test-cluster-enterprise-rocksdb-linux': { testStep('enterprise', 'linux', 'cluster', 'rocksdb', false) }, - if (buildLinux) { - node(os) { - echo "Running " + mode + " " + edition + " " + engine + " " + os + " test" + 'test-singleserver-enterprise-mmfiles-linux': { testStep('enterprise', 'linux', 'singleserver', 'mmfiles', false) }, + 'test-singleserver-enterprise-rocksdb-linux': { testStep('enterprise', 'linux', 'singleserver', 'rocksdb', true) }, - unstashBinaries(edition, os) - testEdition(edition, os, mode, engine) - } - } - else { - echo "Not building " + os + " version" - } - }, + 'test-resilience-community-rocksdb': { testResilienceStep('linux', 'community', 'rocksdb', false) }, + 'test-resilience-community-mmfiles': { testResilienceStep('linux', 'community', 'mmfiles', false) }, - 'test-singleserver-enterprise-mmfiles': { - def mode = 'singleserver' - def edition = 'enterprise' - def engine = 'mmfiles' - - if (buildLinux) { - if (buildEnterprise) { - node(os) { - echo "Running " + mode + " " + edition + " " + engine + " " + os + " test" - - unstashBinaries(edition, os) - testEdition(edition, os, mode, engine) - } - } - else { - echo "Enterprise version not built, skipping 'test-singleserver-enterprise'" - } - } - else { - echo "Not building " + os + " version" - } - }, - - 'test-cluster-community-mmfiles': { - def mode = 'cluster' - def edition = 'community' - def engine = 'mmfiles' - - if (buildLinux) { - node(os) { - echo "Running " + mode + " " + edition + " " + engine + " " + os + " test" - - unstashBinaries(edition, os) - testEdition(edition, os, mode, engine) - } - } - else { - echo "Not building " + os + " version" - } - }, - - 'test-cluster-enterprise-rocksdb': { - def mode = 'cluster' - def edition = 'enterprise' - def engine = 'rocksdb' - - if (buildLinux) { - if (buildEnterprise) { - node(os) { - echo "Running " + mode + " " + edition + " " + engine + " " + os + " test" - - unstashBinaries(edition, os) - testEdition(edition, os, mode, engine) - } - } - else { - echo "Enterprise version not built, skipping 'test-singleserver-enterprise'" - } - } - else { - echo "Not building " + os + " version" - } - }, - - 'jslint': { - if (buildLinux) { - node(os) { - echo "Running jslint test" - - unstashBinaries('community', os) - jslint() - } - } - else { - echo "Not building " + os + " version" - } - } - ) -} - -stage('build other') { - parallel( - 'build-community-windows': { - def os = 'windows' - def edition = 'community' - - if (buildWindows) { - node('windows') { - unstashSourceCode(os) - buildEdition(edition, os) - } - } - else { - echo "Not building " + os + " version" - } - }, - - 'build-community-mac': { - def os = 'mac' - def edition = 'community' - - if (buildMac) { - node('mac') { - unstashSourceCode(os) - buildEdition(edition, os) - } - } - else { - echo "Not building " + os + " version" - } - } + 'jslint': { jslintStep() } ) } diff --git a/Installation/Pipeline/include/build_EDITION_OS.inc b/Installation/Pipeline/include/build_EDITION_OS.inc index 7e1ecbeb87..43cd4d07dd 100644 --- a/Installation/Pipeline/include/build_EDITION_OS.inc +++ b/Installation/Pipeline/include/build_EDITION_OS.inc @@ -1,5 +1,6 @@ #!/bin/bash +load=32 concurrency=$1 edition=$2 os=$3 @@ -32,6 +33,7 @@ echo "PWD: `pwd`" rm -rf log-output/$type.log mkdir -p log-output +touch log-output/$type.log ( mkdir -p build-$edition @@ -46,10 +48,28 @@ mkdir -p log-output -DUSE_FAILURE_TESTS=On \ -DDEBUG_SYNC_REPLICATION=On \ $ENTERPRISE \ - .. > ../log-output/$type.log 2>&1 || exit 1 + .. >> ../log-output/$type.log 2>&1 + + if [ "$?" != 0 ]; then + if fgrep 'Re-run cmake with a different source directory' ../log-output/$type.log; then + rm -rf * + + CXXFLAGS=-fno-omit-frame-pointer \ + cmake \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DUSE_MAINTAINER_MODE=On \ + -DUSE_CATCH_TESTS=On \ + -DUSE_FAILURE_TESTS=On \ + -DDEBUG_SYNC_REPLICATION=On \ + $ENTERPRISE \ + .. >> ../log-output/$type.log 2>&1 || exit 1 + else + exit 1 + fi + fi echo "`date +%T` building..." - make -j$concurrency >> ../log-output/$type.log 2>&1 || exit 1 + make -j $concurrency -l $load >> ../log-output/$type.log 2>&1 || exit 1 ) || exit 1 # copy binaries to preserve them