1
0
Fork 0

Feature/jenkins pipeline (#3327)

This commit is contained in:
Frank Celler 2017-09-26 15:17:47 +02:00 committed by GitHub
parent 01cdaed684
commit b8e999e8da
3 changed files with 306 additions and 202 deletions

View File

@ -118,6 +118,7 @@ resultsKeys = []
resultsStart = [:]
resultsStop = [:]
resultsStatus = [:]
resultsLink = [:]
// -----------------------------------------------------------------------------
// --SECTION-- CONSTANTS AND HELPERS
@ -287,6 +288,17 @@ def getStartPort(os) {
}
}
def releaseStartPort(os, port) {
if (port != 0) {
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
}
}
}
def rspecify(os, test) {
if (os == "windows") {
return [test, test, "--rspec C:\\tools\\ruby23\\bin\\rspec.bat"]
@ -307,29 +319,99 @@ def shellAndPipe(command, logfile) {
sh "(echo 1 > \"${logfile}.result\" ; ${command} ; echo \$? > \"${logfile}.result\") 2>&1 | tee -a \"${logfile}\" ; exit `cat \"${logfile}.result\"`"
}
def logStartStage(logFile) {
def logStartStage(os, logFile, link) {
resultsKeys << logFile
resultsStart[logFile] = new Date()
resultsLink[logFile] = link
resultsStatus[logFile] = "started"
echo "started ${logFile}: ${resultsStart[logFile]}"
if (os == "linux") {
sh "echo 'started ${logFile}: ${resultsStart[logFile]}' | tee -a ${logFile}"
}
generateResult()
}
def logStopStage(logFile) {
def logStopStage(os, logFile) {
resultsStop[logFile] = new Date()
resultsStatus[logFile] = "finished"
echo "finished ${logFile}: ${resultsStop[logFile]}"
if (os == "linux") {
sh "echo 'finished ${logFile}: ${resultsStop[logFile]}' | tee -a ${logFile}"
}
generateResult()
}
def logExceptionStage(logFile, exc) {
def logExceptionStage(os, logFile, exc) {
def msg = exc.toString()
resultsStop[logFile] = new Date()
resultsStatus[logFile] = "failed ${msg}"
echo "failed ${logFile}: ${resultsStop[logFile]} ${msg}"
if (os == "linux") {
sh "echo 'failed ${logFile}: ${resultsStart[logFile]} ${msg}' | tee -a ${logFile}"
}
generateResult()
}
def generateResult() {
def results = ""
def html = "<html><body><table>\n"
html += "<tr><th>Name</th><th>Start</th><th>Stop</th><th>Duration</th><th>Message</th></tr>\n"
for (key in resultsKeys) {
def start = resultsStart[key] ?: ""
def stop = resultsStop[key] ?: ""
def msg = resultsStatus[key] ?: ""
def link = resultsLink[key] ?: ""
if (start != "" && stop == "") {
stop = new Date()
}
def diff = (start != "" && stop != "") ? groovy.time.TimeCategory.minus(stop, start) : "-"
def startf = start == "" ? "-" : start.format('yyyy/MM/dd HH:mm:ss')
def stopf = stop == "" ? "-" : stop.format('yyyy/MM/dd HH:mm:ss')
def color = 'bgcolor="#FF8080"'
def la = ""
def lb = ""
if (link != null) {
la = "<a href=\"$link\">"
lb = "</a>"
}
if (msg == "finished") {
color = 'bgcolor="#80FF80"'
}
else if (msg == "started") {
color = 'bgcolor="#8080FF"'
la = ""
lb = ""
}
results += "${key}: ${startf} - ${stopf} (${diff}) ${msg}\n"
html += "<tr ${color}><td>${la}${key}${lb}</td><td>${startf}</td><td>${stopf}</td><td align=\"right\">${diff}</td><td align=\"right\">${msg}</td></tr>\n"
}
html += "</table></body></html>\n"
node("master") {
fileOperations([fileCreateOperation(fileContent: results, fileName: "results.txt")])
fileOperations([fileCreateOperation(fileContent: html, fileName: "results.html")])
archiveArtifacts(allowEmptyArchive: true, artifacts: "results.*")
}
}
// -----------------------------------------------------------------------------
@ -572,6 +654,42 @@ Running Tests: ${runTests}
// --SECTION-- SCRIPTS STASH
// -----------------------------------------------------------------------------
def stashBuild(os, edition, maintainer) {
lock("stashing-${os}-${edition}-${maintainer}") {
if (os == 'linux' || os == 'mac') {
def name = "build.tar.gz"
sh "rm -f ${name}"
sh "GZIP=-1 tar cpzf ${name} build"
sh "scp ${name} c1:/vol/cache/build-${os}-${edition}-${maintainer}.tar.gz"
}
else if (os == 'windows') {
def name = "build.zip"
bat "del /F /Q ${name}"
powershell "7z a ${name} -r -bd -mx=1 build"
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk ${name} jenkins@c1:/vol/cache/build-${os}-${edition}-${maintainer}.zip"
}
}
}
def unstashBuild(os, edition, maintainer) {
lock("stashing-${os}-${edition}-${maintainer}") {
try {
if (os == "windows") {
powershell "echo 'y' | pscp -i C:\\Users\\Jenkins\\.ssh\\putty-jenkins.ppk jenkins@c1:/vol/cache/build-${os}-${edition}-${maintainer}.zip build.zip"
powershell "Expand-Archive -Path build.zip -Force -DestinationPath ."
}
else {
sh "scp c1:/vol/cache/build-${os}-${edition}-${maintainer}.tar.gz build.tar.gz"
sh "tar xpzf build.tar.gz"
}
}
catch (exc) {
}
}
}
def stashBinaries(os, edition, maintainer) {
def paths = ["build/etc", "etc", "Installation/Pipeline", "js", "scripts", "UnitTests"]
@ -636,15 +754,15 @@ def jslint(os, edition, maintainer) {
def logFile = "${arch}/jslint.log"
try {
logStartStage(logFile)
logStartStage(os, logFile, logFile)
shellAndPipe("./Installation/Pipeline/test_jslint.sh",logFile)
sh "if grep ERROR ${logFile}; then exit 1; fi"
logStopStage(logFile)
logStopStage(os, logFile)
}
catch (exc) {
logExceptionStage(logFile, exc)
logExceptionStage(os, logFile, exc)
renameFolder(arch, archFail)
fileOperations([fileCreateOperation(fileContent: 'JSLINT FAILED', fileName: "${archDir}-FAIL.txt")])
@ -743,133 +861,166 @@ def setupTestEnvironment(os, edition, maintainer, logFile, runDir) {
}
}
def executeTests(os, edition, maintainer, mode, engine, portInit, archDir, arch, stageName) {
def parallelity = (mode == "cluster") ? ((os == "linux") ? 5 : 2) : ((os == "linux") ? 10 : 4)
def singleTest(os, edition, maintainer, mode, engine, test, testArgs, testIndex, stageName, name, port) {
return {
def portInterval = 40
stage("${stageName}-${name}") {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/03-test-${mode}-${engine}"
def archFail = "${arch}-FAIL"
def archRun = "${arch}-RUN"
def logFile = pwd() + "/" + "${arch}/${name}.log"
def logFileRel = "${arch}/${name}.log"
def logFileFailed = pwd() + "/" + "${arch}-FAIL/${name}.log"
def runDir = "run.${testIndex}"
logStartStage(os, logFileRel, logFileRel)
try {
// setup links
setupTestEnvironment(os, edition, maintainer, logFile, runDir)
// assemble command
def command = "./build/bin/arangosh " +
"-c etc/jenkins/arangosh.conf " +
"--log.level warning " +
"--javascript.execute UnitTests/unittest.js " +
"${test} -- " +
"${testArgs} " +
"--minPort " + (port + testIndex * portInterval) + " " +
"--maxPort " + (port + (testIndex + 1) * portInterval - 1)
// 30 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to
// finish within 60 minutes. Even if the features are green
// this is completely broken performance wise...
// DO NOT INCREASE!!
timeout(os == 'linux' ? 30 : 60) {
def tmpDir = pwd() + "/" + runDir + "/tmp"
withEnv(["TMPDIR=${tmpDir}", "TEMPDIR=${tmpDir}", "TMP=${tmpDir}"]) {
if (os == "windows") {
def hostname = powershell(returnStdout: true, script: "hostname")
echo "executing ${command} on ${hostname}"
powershell "cd ${runDir} ; ${command} | Add-Content -PassThru ${logFile}"
}
else {
sh "echo \"Host: `hostname`\" | tee -a ${logFile}"
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
sh "echo \"Date: `date`\" | tee -a ${logFile}"
shellAndPipe("cd ${runDir} ; ./build/bin/arangosh --version", logFile)
command = "(cd ${runDir} ; ${command})"
echo "executing ${command}"
shellAndPipe(command, logFile)
}
}
}
checkCores(os, runDir)
logStopStage(os, logFileRel)
}
catch (exc) {
logExceptionStage(os, logFileRel, exc)
def msg = exc.toString()
echo "caught error, copying log to ${logFileFailed}: ${msg}"
fileOperations([
fileCreateOperation(fileContent: "TEST FAILED: ${msg}", fileName: "${archDir}-FAIL.txt")
])
if (os == 'linux' || os == 'mac') {
sh "echo \"${msg}\" >> ${logFile}"
}
else {
powershell "echo \"${msg}\" | Out-File -filepath ${logFile} -append"
}
copyFile(os, logFile, logFileFailed)
throw exc
}
finally {
def logFileFailedRel = "${arch}-FAIL/${name}.log"
saveCores(os, runDir, name, archRun)
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archDir}-FAIL.txt, ${archRun}/**, ${logFileRel}, ${logFileFailedRel}",
defaultExcludes: false
}
}
}
}
def executeTests(os, edition, maintainer, mode, engine, stageName) {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/03-test-${mode}-${engine}"
def archFail = "${arch}-FAIL"
def archRun = "${arch}-RUN"
def testIndex = 0
def tests = getTests(os, edition, maintainer, mode, engine)
def portInterval = (mode == "cluster") ? 40 : 10
node(testJenkins[os]) {
// this is an `Array.reduce()` in groovy :S
def testSteps = tests.inject([:]) { testMap, testStruct ->
def lockIndex = testIndex % parallelity
def currentIndex = testIndex
// clean the current workspace completely
deleteDirDocker(os)
testIndex++
// create directories for the artifacts
fileOperations([
fileDeleteOperation(excludes: '', includes: "${archDir}-*"),
folderCreateOperation(arch),
folderCreateOperation(archFail),
folderCreateOperation(archRun)
])
def name = testStruct[0]
def test = testStruct[1]
def testArgs = "--prefix ${os}-${edition}-${mode}-${engine} " +
"--configDir etc/jenkins " +
"--skipLogAnalysis true " +
"--skipTimeCritical true " +
"--skipNondeterministic true " +
"--storageEngine ${engine} " +
testStruct[2]
// unstash binaries
unstashBinaries(os, edition, maintainer)
if (mode == "cluster") {
testArgs += " --cluster true"
}
// find a suitable port
def port = (getStartPort(os) as Integer)
echo "Using start port: ${port}"
testMap["${stageName}-${name}"] = {
def logFile = pwd() + "/" + "${arch}/${name}.log"
def logFileRel = "${arch}/${name}.log"
def logFileFailed = pwd() + "/" + "${arch}-FAIL/${name}.log"
def archRun = pwd() + "/" + "${arch}-RUN"
try {
// this is an `Array.reduce()` in groovy :S
def testSteps = tests.inject([:]) { testMap, testStruct ->
def name = testStruct[0]
def test = testStruct[1]
def testArgs = "--prefix ${os}-${edition}-${mode}-${engine} " +
"--configDir etc/jenkins " +
"--skipLogAnalysis true " +
"--skipTimeCritical true " +
"--skipNondeterministic true " +
"--storageEngine ${engine} " +
testStruct[2]
def runDir = "run.${currentIndex}"
def port = portInit + currentIndex * portInterval
testArgs += " --minPort " + port
testArgs += " --maxPort " + (port + portInterval - 1)
def command = "./build/bin/arangosh " +
"-c etc/jenkins/arangosh.conf " +
"--log.level warning " +
"--javascript.execute UnitTests/unittest.js " +
" ${test} -- " +
testArgs
try {
lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${maintainer}-${engine}-${lockIndex}") {
setupTestEnvironment(os, edition, maintainer, logFile, runDir)
try {
logStartStage(logFileRel)
// seriously...30 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
// even if the features are green this is completely broken performance wise..
// DO NOT INCREASE!!
timeout(os == 'linux' ? 30 : 60) {
def tmpDir = pwd() + "/" + runDir + "/tmp"
withEnv(["TMPDIR=${tmpDir}", "TEMPDIR=${tmpDir}", "TMP=${tmpDir}"]) {
if (os == "windows") {
def hostname = powershell(returnStdout: true, script: "hostname")
echo "executing ${command} on ${hostname}"
powershell "cd ${runDir} ; ${command} | Add-Content -PassThru ${logFile}"
}
else {
sh "echo \"Host: `hostname`\" | tee -a ${logFile}"
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
sh "echo \"Date: `date`\" | tee -a ${logFile}"
shellAndPipe("cd ${runDir} ; ./build/bin/arangosh --version", logFile)
command = "(cd ${runDir} ; ${command})"
echo "executing ${command}"
shellAndPipe(command, logFile)
}
}
}
checkCores(os, runDir)
logStopStage(logFileRel)
}
catch (exc) {
logExceptionStage(logFileRel, exc)
def msg = exc.toString()
echo "caught error, copying log to ${logFileFailed}: ${msg}"
fileOperations([
fileCreateOperation(fileContent: "TEST FAILED: ${msg}", fileName: "${archDir}-FAIL.txt")
])
if (os == 'linux' || os == 'mac') {
sh "echo \"${msg}\" >> ${logFile}"
}
else {
powershell "echo \"${msg}\" | Out-File -filepath ${logFile} -append"
}
copyFile(os, logFile, logFileFailed)
throw exc
}
finally {
def logFileFailedRel = "${arch}-FAIL/${name}.log"
saveCores(os, runDir, name, archRun)
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archDir}-FAIL.txt, ${logFileRel}, ${logFileFailedRel}",
defaultExcludes: false
}
if (mode == "cluster") {
testArgs += " --cluster true"
}
testIndex++
testMap["${stageName}-${name}"] = singleTest(os, edition, maintainer, mode, engine, test, testArgs, testIndex, stageName, name, port)
return testMap
}
catch (exc) {
error "test ${name} failed"
}
// fire all tests
parallel testSteps
}
finally {
releaseStartPort(os, port)
}
testMap
}
parallel testSteps
}
def testCheck(os, edition, maintainer, mode, engine) {
@ -900,50 +1051,7 @@ def testCheck(os, edition, maintainer, mode, engine) {
def testStep(os, edition, maintainer, mode, engine, stageName) {
return {
if (testCheck(os, edition, maintainer, mode, engine)) {
node(testJenkins[os]) {
stage(stageName) {
def archDir = "${os}-${edition}-${maintainer}"
def arch = "${archDir}/03-test-${mode}-${engine}"
def archFail = "${arch}-FAIL"
def archRun = "${arch}-RUN"
// clean the current workspace completely
deleteDirDocker(os)
// create directories for the artifacts
fileOperations([
fileDeleteOperation(excludes: '', includes: "${archDir}-*"),
folderCreateOperation(arch),
folderCreateOperation(archFail),
folderCreateOperation(archRun)
])
// unstash binaries
unstashBinaries(os, edition, maintainer)
// find a suitable port
def port = (getStartPort(os) as Integer)
echo "Using start port: ${port}"
try {
executeTests(os, edition, maintainer, mode, engine, port, archDir, arch, stageName)
}
finally {
// release the port reservation
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
}
// archive all artifacts
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archRun}/**",
defaultExcludes: false
}
}
}
executeTests(os, edition, maintainer, mode, engine, stageName)
}
}
}
@ -961,12 +1069,12 @@ def testStepParallel(os, edition, maintainer, modeList) {
def name = "test-${os}-${edition}-${maintainer}"
try {
logStartStage(name)
logStartStage(null, name, null)
parallel branches
logStopStage(name)
logStopStage(null, name)
}
catch (exc) {
logExceptionStage(name, exc)
logExceptionStage(null, name, exc)
throw exc
}
}
@ -1114,9 +1222,13 @@ def buildEdition(os, edition, maintainer) {
def logFile = "${arch}/build.log"
try {
logStartStage(logFile)
logStartStage(os, logFile, logFile)
if (os == 'linux' || os == 'mac') {
if (! fileExists('build/Makefile')) {
unstashBuild(os, edition, maintainer)
}
sh "echo \"Host: `hostname`\" | tee -a ${logFile}"
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
sh "echo \"Date: `date`\" | tee -a ${logFile}"
@ -1147,10 +1259,10 @@ def buildEdition(os, edition, maintainer) {
powershell ". .\\Installation\\Pipeline\\windows\\build_${os}_${edition}_${maintainer}.ps1"
}
logStopStage(logFile)
logStopStage(os, logFile)
}
catch (exc) {
logExceptionStage(logFile, exc)
logExceptionStage(os, logFile, exc)
def msg = exc.toString()
@ -1169,6 +1281,10 @@ def buildEdition(os, edition, maintainer) {
throw exc
}
finally {
if (os == "linux") {
stashBuild(os, edition, maintainer)
}
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archDir}-FAIL.txt, ${arch}/**, ${archFail}/**",
defaultExcludes: false
@ -1234,16 +1350,16 @@ def createDockerImage(edition, maintainer, stageName) {
withEnv(["DOCKERTAG=${packageName}-${dockerTag}"]) {
try {
logStartStage(logFile)
logStartStage(os, logFile, logFile)
shellAndPipe("./scripts/build-docker.sh", logFile)
shellAndPipe("docker tag arangodb:${packageName}-${dockerTag} c1.triagens-gmbh.zz:5000/arangodb/${packageName}:${dockerTag}", logFile)
shellAndPipe("docker push c1.triagens-gmbh.zz:5000/arangodb/${packageName}:${dockerTag}", logFile)
logStopStage(logFile)
logStopStage(os, logFile)
}
catch (exc) {
logExceptionStage(logFile, exc)
logExceptionStage(os, logFile, exc)
renameFolder(arch, archFail)
fileOperations([fileCreateOperation(fileContent: 'DOCKER FAILED', fileName: "${archDir}-FAIL.txt")])
@ -1349,34 +1465,6 @@ timestamps {
runOperatingSystems(['linux', 'mac', 'windows'])
}
finally {
results = ""
html = "<html><body><table>\n"
html += "<tr><th>Name</th><th>Start</th><th>Stop</th><th>Duration</th><th>Message</th></tr>\n"
for (key in resultsKeys) {
def start = resultsStart[key] ?: ""
def stop = resultsStop[key] ?: ""
def msg = resultsStatus[key] ?: ""
def diff = (start != "" && stop != "") ? groovy.time.TimeCategory.minus(stop, start) : "-"
def startf = start.format('yyyy/MM/dd HH:mm:ss')
def stopf = stop.format('yyyy/MM/dd HH:mm:ss')
def color = 'bgcolor="#FF8080"'
if (msg == "finished") {
color = 'bgcolor="#80FF80"'
}
results += "${key}: ${startf} - ${stopf} (${diff}) ${msg}\n"
html += "<tr ${color}><td>${key}</td><td>${startf}</td><td>${stopf}</td><td align=\"right\">${diff}</td><td align=\"right\">${msg}</td></tr>\n"
}
html += "</table></body></html>\n"
node("master") {
fileOperations([fileCreateOperation(fileContent: results, fileName: "results.txt")])
fileOperations([fileCreateOperation(fileContent: html, fileName: "results.html")])
archiveArtifacts(allowEmptyArchive: true, artifacts: "results.*")
}
generateResult()
}
}

View File

@ -48,6 +48,20 @@ fi
mkdir -p build
if [ ! -f build/location ]; then
if [ "$os" == mac ]; then
(ls -l && echo "$os-$edition-$maintainer") | md5 | awk '{print $1}' > build/location
else
(ls -l && echo "$os-$edition-$maintainer") | md5sum | awk '{print $1}' > build/location
fi
fi
GENPATH="/tmp/`cat build/location`"
rm -f $GENPATH
ln -s `pwd` $GENPATH
cd $GENPATH
if [ -z "$logdir" ]; then
logdir=log-output
rm -rf $logdir

View File

@ -0,0 +1,2 @@
[server]
authentication = false