Hawkin
/
arangodb
mirror of https://gitee.com/bigwinds/arangodb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Feature/jenkins pipeline (#3241)

This commit is contained in:
Frank Celler 2017-09-12 12:03:46 +02:00 committed by GitHub
parent 7c3215c386
commit 66fdaa24a5
8 changed files with 346 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

439
Installation/Pipeline/Jenkinsfile.groovy
View File

@ -22,6 +22,8 @@ def defaultBuild = true
def defaultCleanBuild = false
def defaultCommunity = true
def defaultEnterprise = true
def defaultMaintainer = true
def defaultUser = false
// def defaultRunResilience = false
def defaultRunTests = true
@ -57,6 +59,16 @@ properties([
description: 'build and run tests for enterprise',
name: 'Enterprise'
),
booleanParam(
defaultValue: defaultMaintainer,
description: 'build in maintainer mode',
name: 'Maintainer'
),
booleanParam(
defaultValue: defaultUser,
description: 'build in user (aka non-maintainer) mode',
name: 'User'
),
// booleanParam(
// defaultValue: defaultRunResilience,
// description: 'run resilience tests',
@ -73,12 +85,6 @@ properties([
// start with empty build directory
cleanBuild = params.cleanBuild
// build community
useCommunity = params.Community
// build enterprise
useEnterprise = params.Enterprise
// build linux
useLinux = params.Linux
@ -88,6 +94,18 @@ useMac = params.Mac
// build windows
useWindows = params.Windows
// build and test community
useCommunity = params.Community
// build and test enterprise
useEnterprise = params.Enterprise
// build maintainer mode
useMaintainer = params.Maintainer
// build user mode
useUser = params.User
// run resilience tests
//runResilience = params.runResilience
@ -97,6 +115,9 @@ runTests = params.runTests
// restrict builds
restrictions = [:]
// overview of configured builds and tests
overview = ""
// -----------------------------------------------------------------------------
// --SECTION-- CONSTANTS AND HELPERS
// -----------------------------------------------------------------------------
@ -150,6 +171,12 @@ def copyFile(os, src, dst) {
}
}
def renameFolder(src, dst) {
fileOperations([
folderRenameOperation(destination: dst, source: src)
])
}
def checkEnabledOS(os, text) {
if (os == 'linux' && ! useLinux) {
echo "Not ${text} ${os} because ${os} is not enabled"
@ -183,35 +210,54 @@ def checkEnabledEdition(edition, text) {
return true
}
def checkCoresAndSave(os, runDir, name, archRuns, archCores) {
def checkEnabledMaintainer(maintainer, os, text) {
if (os == 'windows' && maintainer != 'maintainer') {
echo "Not ${text} ${maintainer} because ${maintainer} is not enabled under Windows"
return false
}
if (maintainer == 'maintainer' && ! useMaintainer) {
echo "Not ${text} ${maintainer} because ${maintainer} is not enabled"
return false
}
if (maintainer == 'user' && ! useUser) {
echo "Not ${text} ${maintainer} because ${maintainer} is not enabled"
return false
}
return true
}
def checkCoresAndSave(os, runDir, name, archRun) {
if (os == 'windows') {
powershell "move-item -Force -ErrorAction Ignore ${runDir}/logs ${archRuns}/${name}.logs"
powershell "move-item -Force -ErrorAction Ignore ${runDir}/out ${archRuns}/${name}.logs"
powershell "move-item -Force -ErrorAction Ignore ${runDir}/tmp ${archRuns}/${name}.tmp"
powershell "move-item -Force -ErrorAction Ignore ${runDir}/logs ${archRun}/${name}.logs"
powershell "move-item -Force -ErrorAction Ignore ${runDir}/out ${archRun}/${name}.logs"
powershell "move-item -Force -ErrorAction Ignore ${runDir}/tmp ${archRun}/${name}.tmp"
def files = findFiles(glob: "${runDir}/*.dmp")
if (files.length > 0) {
for (file in files) {
powershell "move-item -Force -ErrorAction Ignore ${file} ${archCores}"
powershell "move-item -Force -ErrorAction Ignore ${file} ${archRun}"
}
powershell "copy-item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${archCores}"
powershell "copy-item .\\build\\bin\\* -Include *.exe,*.pdb,*.ilk ${archRun}"
error("found dmp file")
}
}
else {
sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"
sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRun}/${name}.\$i\" || true; done"
def files = findFiles(glob: '${runDir}/core*')
if (files.length > 0) {
for (file in files) {
sh "mv ${file} ${archCores}"
sh "mv ${file} ${archRun}"
}
sh "cp -a build/bin/* ${archCores}"
sh "cp -a build/bin/* ${archRun}"
error("found core file")
}
@ -361,9 +407,14 @@ def checkCommitMessages() {
else if (skip) {
useLinux = false
useMac = false
useWindows = false
useCommunity = false
useEnterprise = false
useMaintainer = false
useUser = false
// runResilience = false
runTests = false
}
@ -375,56 +426,79 @@ def checkCommitMessages() {
echo "build of PR"
restrictions = [
"build-community-linux" : true,
"build-community-mac" : true,
"build-community-windows" : true,
"build-enterprise-linux" : true,
"build-enterprise-mac" : true,
"build-enterprise-windows" : true,
"test-cluster-community-mmfiles-linux" : true,
"test-cluster-community-rocksdb-linux" : true,
"test-cluster-enterprise-mmfiles-linux" : true,
"test-cluster-enterprise-rocksdb-linux" : true,
"test-singleserver-community-mmfiles-linux" : true,
"test-singleserver-community-rocksdb-linux" : true,
"test-singleserver-enterprise-mmfiles-linux" : true,
"test-singleserver-enterprise-rocksdb-linux" : true
// OS EDITION MAINTAINER
"build-linux-community-maintainer" : true,
"build-linux-enterprise-maintainer" : true,
"build-mac-community-maintainer" : true,
"build-mac-enterprise-user" : true,
"build-windows-community-maintainer" : true,
"build-windows-enterprise-maintainer" : true,
// OS EDITION MAINTAINER MODE ENGINE
"test-linux-enterprise-maintainer-cluster-rocksdb" : true,
"test-linux-community-maintainer-singleserver-mmfiles" : true
]
}
else {
restrictions = [
"build-community-mac" : true,
"build-enterprise-linux" : true,
"test-cluster-enterprise-rocksdb-linux" : true,
"test-singleserver-enterprise-mmfiles-linux" : true
// OS EDITION MAINTAINER
"build-linux-community-user" : true,
"build-linux-enterprise-maintainer" : true,
// OS EDITION MAINTAINER MODE ENGINE
"test-linux-enterprise-maintainer-cluster-rocksdb" : true,
"test-linux-community-user-singleserver-mmfiles" : true
]
}
}
echo """BRANCH_NAME: ${env.BRANCH_NAME}
overview = """BRANCH_NAME: ${env.BRANCH_NAME}
SOURCE: ${sourceBranchLabel}
CHANGE_ID: ${env.CHANGE_ID}
CHANGE_TARGET: ${env.CHANGE_TARGET}
JOB_NAME: ${env.JOB_NAME}
CAUSE: ${causeDescription}
"""
Linux: ${useLinux}
if (restrictions) {
useLinux = true
useMac = true
useWindows = true
useCommunity = true
useEnterprise = true
useMaintainer = true
useUser = true
// runResilience = true
runTests = true
overview += "Restrictions:\n"
for (r in restrictions.keySet()) {
overview += " " + r + "\n"
}
}
else {
overview += """Linux: ${useLinux}
Mac: ${useMac}
Windows: ${useWindows}
Clean Build: ${cleanBuild}
Building Community: ${useCommunity}
Building Enterprise: ${useEnterprise}
Building Maintainer: ${useMaintainer}
Building Non-Maintainer: ${useUser}
Running Tests: ${runTests}
Restrictions: ${restrictions.keySet().join(", ")}
"""
}
}
// -----------------------------------------------------------------------------
// --SECTION-- SCRIPTS STASH
// -----------------------------------------------------------------------------
def stashBinaries(os, edition) {
def stashBinaries(os, edition, maintainer) {
def paths = ["build/etc", "etc", "Installation/Pipeline", "js", "scripts", "UnitTests"]
if (edition == "enterprise") {
@ -445,25 +519,25 @@ def stashBinaries(os, edition) {
// this is a super mega mess...scp will run as the system user and not as jenkins when run as a server
// I couldn't figure out how to properly get it running for hours...so last resort was to install putty
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk stash.zip jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.zip"
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk stash.zip jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.zip"
}
else {
paths << "build/bin/"
paths << "build/tests/"
sh "GZIP=-1 tar cpzf stash.tar.gz " + paths.join(" ")
sh "scp stash.tar.gz c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.tar.gz"
sh "scp stash.tar.gz c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.tar.gz"
}
}
def unstashBinaries(os, edition) {
def unstashBinaries(os, edition, maintainer) {
if (os == "windows") {
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.zip stash.zip"
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk jenkins@c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.zip stash.zip"
powershell "Expand-Archive -Path stash.zip -Force -DestinationPath ."
powershell "copy build\\bin\\RelWithDebInfo\\* build\\bin"
}
else {
sh "scp c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}.tar.gz stash.tar.gz"
sh "scp c1:/vol/cache/binaries-${env.BUILD_TAG}-${os}-${edition}-${maintainer}.tar.gz stash.tar.gz"
sh "tar xpzf stash.tar.gz"
}
}
@ -472,15 +546,40 @@ def unstashBinaries(os, edition) {
// --SECTION-- SCRIPTS JSLINT
// -----------------------------------------------------------------------------
def jslint() {
sh './Installation/Pipeline/test_jslint.sh'
def jslint(os, edition, maintainer) {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/02-jslint"
def archFail = "${archDir}/02-jslint-FAIL"
fileOperations([
folderDeleteOperation(arch),
folderDeleteOperation(archFail),
folderCreateOperation(arch)
])
def logFile = "${arch}/jslint.log"
try {
sh "./Installation/Pipeline/test_jslint.sh | tee ${logFile}"
sh "if grep ERROR ${logFile}; then exit 1; fi"
}
catch (exc) {
renameFolder(arch, archFail)
fileOperations([fileCreateOperation(fileContent: 'BUILD FAILED', fileName: "${arch}-FAIL.txt")])
throw exc
}
finally {
archiveArtifacts allowEmptyArchive: true,
artifacts: "${arch}-*, ${arch}/**, ${archFail}/**",
defaultExcludes: false
}
}
// -----------------------------------------------------------------------------
// --SECTION-- SCRIPTS TESTS
// -----------------------------------------------------------------------------
def getTests(os, edition, mode, engine) {
def getTests(os, edition, maintainer, mode, engine) {
def tests = [
["arangobench", "arangobench" , ""],
["arangosh", "arangosh", "--skipShebang true"],
@ -525,7 +624,7 @@ def getTests(os, edition, mode, engine) {
return tests
}
def setupTestEnvironment(os, edition, logFile, runDir) {
def setupTestEnvironment(os, edition, maintainer, logFile, runDir) {
fileOperations([
folderCreateOperation("${runDir}/tmp"),
])
@ -550,10 +649,10 @@ def setupTestEnvironment(os, edition, logFile, runDir) {
}
}
def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed, archCores) {
def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch, stageName) {
def parallelity = 4
def testIndex = 0
def tests = getTests(os, edition, mode, engine)
def tests = getTests(os, edition, maintainer, mode, engine)
def portInterval = (mode == "cluster") ? 40 : 10
@ -578,11 +677,13 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
testArgs += " --cluster true"
}
testMap["test-${os}-${edition}-${mode}-${engine}-${name}"] = {
def logFile = "${arch}/${name}.log"
def logFileFailed = "${archFailed}/${name}.log"
testMap["${stageName}-${name}"] = {
def logFile = pwd() + "/" + "${arch}/${name}.log"
def logFileFailed = pwd() + "/" + "${arch}-FAIL/${name}.log"
def archRun = pwd() + "/" + "${arch}-RUN"
def runDir = "run.${currentIndex}"
def port = portInit + currentIndex * portInterval
def port = portInit + currentIndex * portInterval
testArgs += " --minPort " + port
testArgs += " --maxPort " + (port + portInterval - 1)
@ -594,10 +695,15 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
testArgs
try {
lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${engine}-${lockIndex}") {
setupTestEnvironment(os, edition, logFile, runDir)
lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${maintainer}-${engine}-${lockIndex}") {
setupTestEnvironment(os, edition, maintainer, logFile, runDir)
try {
// seriously...30 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
// even if the features are green this is completely broken performance wise..
// DO NOT INCREASE!!
timeout(30) {
def tmpDir = pwd() + "/" + runDir + "/tmp"
@ -622,11 +728,23 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
catch (exc) {
echo "caught error, copying log to ${logFileFailed}"
echo exc.toString()
fileOperations([
fileCreateOperation(fileContent: 'TEST FAILED', fileName: "${archDir}-FAIL.txt")
])
copyFile(os, logFile, logFileFailed)
throw exc
}
finally {
checkCoresAndSave(os, runDir, name, archRuns, archCores)
def logFileRel = "${arch}/${name}.log"
def logFileFailedRel = "${arch}-FAIL/${name}.log"
checkCoresAndSave(os, runDir, name, archRun)
archiveArtifacts allowEmptyArchive: true,
artifacts: "${logFileRel}, ${logFileFailedRel}",
defaultExcludes: false
}
}
}
@ -641,7 +759,12 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
parallel testSteps
}
def testCheck(os, edition, mode, engine) {
def testCheck(os, edition, maintainer, mode, engine) {
if (! runTests) {
echo "Not testing ${os} ${mode} because testing is not enabled"
return false
}
if (! checkEnabledOS(os, 'testing')) {
return false
}
@ -650,32 +773,26 @@ def testCheck(os, edition, mode, engine) {
return false
}
if (! runTests) {
echo "Not testing ${os} ${mode} because testing is not enabled"
return false
if (! checkEnabledMaintainer(maintainer, os, 'building')) {
return false
}
if (restrictions && !restrictions["test-${mode}-${edition}-${engine}-${os}"]) {
if (restrictions && !restrictions["test-${os}-${edition}-${maintainer}-${mode}-${engine}"]) {
return false
}
return true
}
def testStep(os, edition, mode, engine, testName) {
def testStep(os, edition, maintainer, mode, engine, stageName) {
return {
if (testCheck(os, edition, mode, engine)) {
if (testCheck(os, edition, maintainer, mode, engine)) {
node(testJenkins[os]) {
stage(testName) {
def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
def archFailedRel = "${archRel}_FAILED"
def archRunsRel = "${archRel}_RUN"
def archCoresRel = "${archRel}_CORES"
def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
def archFailed = "${arch}_FAILED"
def archRuns = "${arch}_RUN"
def archCores = "${arch}_CORES"
stage(stageName) {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/03-test-${mode}-${engine}"
def archFail = "${arch}-FAIL"
def archRun = "${arch}-RUN"
// clean the current workspace completely
deleteDir()
@ -683,45 +800,33 @@ def testStep(os, edition, mode, engine, testName) {
// create directories for the artifacts
fileOperations([
folderCreateOperation(arch),
folderCreateOperation(archFailed),
folderCreateOperation(archRuns),
folderCreateOperation(archCores)
folderCreateOperation(archFail),
folderCreateOperation(archRun)
])
// unstash binaries
unstashBinaries(os, edition)
unstashBinaries(os, edition, maintainer)
// find a suitable port
def port = (getStartPort(os) as Integer)
echo "Using start port: ${port}"
// seriously...60 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
// even if the features are green this is completely broken performance wise..
// DO NOT INCREASE!!
timeout(60) {
try {
executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
try {
executeTests(os, edition, maintainer, mode, engine, port, archDir, arch, stageName)
}
finally {
// release the port reservation
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
finally {
// step([$class: 'XUnitBuilder',
// thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
// tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])
// release the port reservation
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
}
// archive all artifacts
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
defaultExcludes: false
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
}
// archive all artifacts
archiveArtifacts allowEmptyArchive: true,
artifacts: "${arch}-*, ${archRun}/**",
defaultExcludes: false
}
}
}
@ -729,13 +834,13 @@ def testStep(os, edition, mode, engine, testName) {
}
}
def testStepParallel(os, edition, modeList) {
def testStepParallel(os, edition, maintainer, modeList) {
def branches = [:]
for (mode in modeList) {
for (engine in ['mmfiles', 'rocksdb']) {
def name = "test-${os}-${edition}-${mode}-${engine}";
branches[name] = testStep(os, edition, mode, engine, name)
def stageName = "test-${os}-${edition}-${maintainer}-${mode}-${engine}";
branches[stageName] = testStep(os, edition, maintainer, mode, engine, stageName)
}
}
@ -870,66 +975,90 @@ def testStepParallel(os, edition, modeList) {
// --SECTION-- SCRIPTS BUILD
// -----------------------------------------------------------------------------
def buildEdition(os, edition) {
def arch = "01_build_${os}_${edition}"
def buildEdition(os, edition, maintainer) {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/01-build"
def archFail = "${archDir}/01-build-FAIL"
fileOperations([
folderDeleteOperation(arch),
fileDeleteOperation(excludes: '', includes: "${archDir}-*"),
folderDeleteOperation(archFail),
folderCreateOperation(arch)
])
def logFile = "${arch}/build.log"
try {
if (os == 'linux') {
sh "./Installation/Pipeline/linux/build_${os}_${edition}.sh 64 ${arch}"
}
else if (os == 'mac') {
sh "./Installation/Pipeline/mac/build_${os}_${edition}.sh 16 ${arch}"
}
else if (os == 'windows') {
// I concede...we need a lock for windows...I could not get it to run concurrently...
// v8 would not build multiple times at the same time on the same machine:
// PDB API call failed, error code '24': ' etc etc
// in theory it should be possible to parallelize it by setting an environment variable
// (see the build script) but for v8 it won't work :(
// feel free to recheck if there is time somewhen...this thing here really should not be possible but
// ensure that there are 2 concurrent builds on the SAME node building v8 at the same time to properly
// test it. I just don't want any more "yeah that might randomly fail. just restart" sentences any more.
if (os == 'linux' || os == 'mac') {
sh "echo \"Host: `hostname`\" | tee ${logFile}"
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
sh "echo \"Date: `date`\" | tee -a ${logFile}"
def hostname = powershell(returnStdout: true, script: "hostname")
lock('build-${hostname}') {
powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}.ps1"
if (os == 'linux') {
sh "./Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh 64 ${os} ${edition} ${maintainer} ${arch}"
}
else if (os == 'mac') {
sh "./Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh 16 ${os} ${edition} ${maintainer} ${arch}"
}
}
else if (os == 'windows') {
def tmpDir = "${arch}/tmp"
fileOperations([
folderCreateOperation(tmpDir)
])
// withEnv(["TMPDIR=${tmpDir}", "TEMPDIR=${tmpDir}", "TMP=${tmpDir}",
// "_MSPDBSRV_ENDPOINT_=${edition}-${env.BUILD_TAG}", "GYP_USE_SEPARATE_MSPDBSRV=1"]) {
powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}.ps1"
// }
fileOperations([
folderDeleteOperation(tmpDir)
])
}
}
catch (exc) {
fileOperations([
fileCreateOperation(fileContent: 'BUILD FAILED', fileName: "${archDir}-FAIL.txt")
])
renameFolder(arch, archFail)
throw exc
}
finally {
archiveArtifacts allowEmptyArchive: true,
artifacts: "${arch}/**",
artifacts: "${archDir}-*, ${arch}/**, ${archFail}/**",
defaultExcludes: false
}
}
def buildStepCheck(os, edition) {
def buildStepCheck(os, edition, maintainer) {
if (! checkEnabledOS(os, 'building')) {
return false
}
if (! checkEnabledEdition(edition, 'testing')) {
if (! checkEnabledEdition(edition, 'building')) {
return false
}
if (restrictions && !restrictions["build-${edition}-${os}"]) {
if (! checkEnabledMaintainer(maintainer, os, 'building')) {
return false
}
if (restrictions && !restrictions["build-${os}-${edition}-${maintainer}"]) {
return false
}
return true
}
def runEdition(os, edition) {
def runEdition(os, edition, maintainer, stageName) {
return {
if (buildStepCheck(os, edition)) {
if (buildStepCheck(os, edition, maintainer)) {
node(buildJenkins[os]) {
stage("build-${os}-${edition}") {
stage(stageName) {
timeout(30) {
checkoutCommunity()
@ -940,9 +1069,30 @@ def runEdition(os, edition) {
// checkoutResilience()
}
timeout(90) {
buildEdition(os, edition)
stashBinaries(os, edition)
// I concede...we need a lock for windows...I could not get it to run concurrently...
// v8 would not build multiple times at the same time on the same machine:
// PDB API call failed, error code '24': ' etc etc
// in theory it should be possible to parallelize it by setting an environment variable
// (see the build script) but for v8 it won't work :(
// feel free to recheck if there is time somewhen...this thing here really should not be possible but
// ensure that there are 2 concurrent builds on the SAME node building v8 at the same time to properly
// test it. I just don't want any more "yeah that might randomly fail. just restart" sentences any more.
if (os == "windows") {
def hostname = powershell(returnStdout: true, script: "hostname").trim()
lock("build-windows-${hostname}") {
timeout(90) {
buildEdition(os, edition, maintainer)
stashBinaries(os, edition, maintainer)
}
}
}
else {
timeout(90) {
buildEdition(os, edition, maintainer)
stashBinaries(os, edition, maintainer)
}
}
}
@ -950,12 +1100,12 @@ def runEdition(os, edition) {
if (os == "linux") {
stage("jslint-${edition}") {
echo "Running jslint for ${edition}"
jslint()
jslint(os, edition, maintainer)
}
}
}
testStepParallel(os, edition, ['cluster', 'singleserver'])
testStepParallel(os, edition, maintainer, ['cluster', 'singleserver'])
}
}
}
@ -969,7 +1119,10 @@ def runOperatingSystems(osList) {
for (os in osList) {
for (edition in ['community', 'enterprise']) {
branches["build-${os}-${edition}"] = runEdition(os, edition)
for (maintainer in ['maintainer', 'user']) {
def stageName = "build-${os}-${edition}-${maintainer}"
branches[stageName] = runEdition(os, edition, maintainer, stageName)
}
}
}
@ -978,5 +1131,11 @@ def runOperatingSystems(osList) {
timestamps {
checkCommitMessages()
node("master") {
fileOperations([fileCreateOperation(fileContent: overview, fileName: "overview.txt")])
archiveArtifacts(allowEmptyArchive: true, artifacts: "overview.txt")
}
runOperatingSystems(['linux', 'mac', 'windows'])
}
}

35
Installation/Pipeline/include/build_OS_EDITION.inc → Installation/Pipeline/build_OS_EDITION_MAINTAINER.sh Normal file → Executable file
View File

@ -1,7 +1,9 @@
#!/bin/bash
concurrency=$1
edition=$2
os=$3
logdir=$4
os=$2
edition=$3
maintainer=$4
logdir=$5
ENTERPRISE=""
type="build"
@ -17,6 +19,19 @@ else
exit 1
fi
MAINTAINER=""
if [ "$maintainer" == maintainer ]; then
MAINTAINER="-DUSE_MAINTAINER_MODE=On"
type="${type}_maintainer"
elif [ "$maintainer" == user ]; then
MAINTAINER="-DUSE_MAINTAINER_MODE=Off"
type="${type}_user"
else
echo "$0: unknown maintainer '$maintainer', expecting 'maintainer' or 'user'"
exit 1
fi
if [ "$os" == linux ]; then
type="${type}_linux"
load=40
@ -28,10 +43,6 @@ else
exit 1
fi
echo "CONCURRENY: $concurrency"
echo "HOST: `hostname`"
echo "PWD: `pwd`"
mkdir -p build
if [ -z "$logdir" ]; then
@ -42,6 +53,10 @@ fi
touch $logdir/build.log
echo "CONCURRENY: $concurrency" | tee -a $logdir/build.log
echo "HOST: `hostname`" | tee -a $logdir/build.log
echo "PWD: `pwd`" | tee -a $logdir/build.log
(
set -eo pipefail
cd build
@ -50,15 +65,15 @@ touch $logdir/build.log
CXXFLAGS=-fno-omit-frame-pointer \
cmake \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DUSE_MAINTAINER_MODE=On \
-DUSE_CATCH_TESTS=On \
-DUSE_FAILURE_TESTS=On \
-DDEBUG_SYNC_REPLICATION=On \
$MAINTAINER \
$ENTERPRISE \
.. 2>&1 | tee ../$logdir/build.log
.. 2>&1 | tee -a ../$logdir/build.log
echo "`date +%T` building..."
make -j $concurrency -l $load 2>&1 | tee -a ../$logdir/build.log
) || exit 1
echo "`date +%T` done..."
echo "`date +%T` done..."

3
Installation/Pipeline/linux/build_linux_community.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" community linux "$2"

3
Installation/Pipeline/linux/build_linux_enterprise.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" enterprise linux "$2"

3
Installation/Pipeline/mac/build_mac_community.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" community mac "$2"

3
Installation/Pipeline/mac/build_mac_enterprise.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
. ./Installation/Pipeline/include/build_OS_EDITION.inc "$1" enterprise mac "$2"

13
Installation/Pipeline/windows/build_windows_community.ps1
View File

@ -1,10 +1,12 @@
$logdir="01_build_windows_community"
$logdir="windows-community-maintainer/01-build"
$ErrorActionPreference="Stop"
$vcpath=$(Get-ItemProperty HKLM:\SOFTWARE\Wow6432Node\Microsoft\VisualStudio\SxS\VC7)."14.0"
#$env:_MSPDBSRV_ENDPOINT_="community-${env:BUILD_TAG}"
$buildOptions = "-DUSE_MAINTAINER_MODE=On -DUSE_ENTERPRISE=Off -DUSE_CATCH_TESTS=On -DUSE_FAILURE_TESTS=On -DDEBUG_SYNC_REPLICATION=On -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSKIP_PACKAGING=On"
Remove-Item -Force -Recurse ${logdir} -ErrorAction SilentlyContinue
New-Item -Force -ItemType Directory ${logdir} -ErrorAction SilentlyContinue
if (Get-Command docker -errorAction SilentlyContinue) {
$buildOptions += " -DOPENSSL_INCLUDE_DIR=`"`$env:OPENSSL_INCLUDE_DIR`" -DLIB_EAY_RELEASE=`"`$env:LIB_EAY_RELEASE`" -DSSL_EAY_RELEASE=`"`$env:SSL_EAY_RELEASE`" -DLIB_EAY_RELEASE_DLL=`"`$env:LIB_EAY_RELEASE_DLL`" -DSSL_EAY_RELEASE_DLL=`"`$env:SSL_EAY_RELEASE_DLL"
$volume = "$env:WORKSPACE"
@ -22,9 +24,16 @@ exit $LastExitCode
docker run --rm -v $volume m0ppers/build-container powershell C:\arangodb\buildscript.ps1 | Set-Content -PassThru ${logdir}\build.log
} else {
$env:GYP_MSVS_OVERRIDE_PATH="${vcpath}\bin"
$env:CC="${vcpath}\bin\cl.exe"
$env:CXX="${vcpath}\bin\cl.exe"
$env
New-Item -ItemType Directory -Force -Path build
cd build
Invoke-Expression "cmake .. -G `"Visual Studio 15 2017 Win64`" ${buildOptions} | Set-Content -PassThru ..\${logdir}\build.log"
cmake --build . --config RelWithDebInfo | Add-Content -PassThru ..\${logdir}\build.log
cd ..
}

13
Installation/Pipeline/windows/build_windows_enterprise.ps1
View File

@ -1,10 +1,12 @@
$logdir="01_build_windows_enterprise"
$logdir="windows-enterprise-maintainer/01-build"
$ErrorActionPreference="Stop"
$vcpath=$(Get-ItemProperty HKLM:\SOFTWARE\Wow6432Node\Microsoft\VisualStudio\SxS\VC7)."14.0"
#$env:_MSPDBSRV_ENDPOINT_="enterprise-${env:BUILD_TAG}"
$buildOptions = "-DUSE_MAINTAINER_MODE=On -DUSE_ENTERPRISE=On -DUSE_CATCH_TESTS=On -DUSE_FAILURE_TESTS=On -DDEBUG_SYNC_REPLICATION=On -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSKIP_PACKAGING=On"
Remove-Item -Force -Recurse ${logdir} -ErrorAction SilentlyContinue
New-Item -Force -ItemType Directory ${logdir} -ErrorAction SilentlyContinue
if (Get-Command docker -errorAction SilentlyContinue) {
$buildOptions += " -DOPENSSL_INCLUDE_DIR=`"`$env:OPENSSL_INCLUDE_DIR`" -DLIB_EAY_RELEASE=`"`$env:LIB_EAY_RELEASE`" -DSSL_EAY_RELEASE=`"`$env:SSL_EAY_RELEASE`" -DLIB_EAY_RELEASE_DLL=`"`$env:LIB_EAY_RELEASE_DLL`" -DSSL_EAY_RELEASE_DLL=`"`$env:SSL_EAY_RELEASE_DLL"
$volume = "$env:WORKSPACE"
@ -22,9 +24,16 @@ exit $LastExitCode
docker run --rm -v $volume m0ppers/build-container powershell C:\arangodb\buildscript.ps1 | Set-Content -PassThru ${logdir}\build.log
} else {
$env:GYP_MSVS_OVERRIDE_PATH="${vcpath}\bin"
$env:CC="${vcpath}\bin\cl.exe"
$env:CXX="${vcpath}\bin\cl.exe"
$env
New-Item -ItemType Directory -Force -Path build
cd build
Invoke-Expression "cmake .. -G `"Visual Studio 15 2017 Win64`" ${buildOptions} | Set-Content -PassThru ..\${logdir}\build.log"
cmake --build . --config RelWithDebInfo | Add-Content -PassThru ..\${logdir}\build.log
cd ..
}