From 80b2db0cd146c3a5352ca5013e9dc28e7ea0b040 Mon Sep 17 00:00:00 2001 From: Dean Allemang Date: Tue, 19 Dec 2017 16:59:27 -0500 Subject: [PATCH 01/15] INFRA-224 --- etc/process/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index c52d6729b..1036b44db 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -82,6 +82,7 @@ pipeline { // If you want to check out a specific version of fibo-infra add the "branch:" parameter // //git branch: 'INFRA-161', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + git branch: 'INFRA-224', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' echo 'Checked out fibo-infra repo' From 3abde4f279bba86c202ef255469f5bbf93a858ff Mon Sep 17 00:00:00 2001 From: Dean Allemang Date: Wed, 20 Dec 2017 10:09:31 -0500 Subject: [PATCH 02/15] INFRA-224 Put in a debug line for the script. Which script am I running? --- etc/process/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 1036b44db..c2f3428b5 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -105,6 +105,7 @@ pipeline { // dir('fibo-infra') { echo 'Execute the publish-fibo.sh script:' + cat './jenkins/bin/publish-fibo.sh init' sh './jenkins/bin/publish-fibo.sh init' echo 'Finished executing the publish-fibo script' } From a0c9a1d65a4fe5f37cc5182102adcad2e44e1a1c Mon Sep 17 00:00:00 2001 From: Dean Allemang Date: Wed, 20 Dec 2017 10:17:41 -0500 Subject: [PATCH 03/15] INFRA-224 Fix Jenkinsfile to get the right branch --- etc/process/Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index c2f3428b5..6f3e86a5b 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -84,7 +84,7 @@ pipeline { //git branch: 'INFRA-161', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' git branch: 'INFRA-224', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' // - git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' echo 'Checked out fibo-infra repo' // // Stash the nomagic templates so that they can be unstashed on the nomagix box @@ -105,7 +105,6 @@ pipeline { // dir('fibo-infra') { echo 'Execute the publish-fibo.sh script:' - cat './jenkins/bin/publish-fibo.sh init' sh './jenkins/bin/publish-fibo.sh init' echo 'Finished executing the publish-fibo script' } From de7b1b92f5df2eb27c0146c2163bb5bf97d06427 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 15:59:11 +0000 Subject: [PATCH 04/15] INFRA-224 Removed nomagic node from Jenkinsfile --- etc/process/Jenkinsfile | 341 +++++++++++----------------------------- 1 file changed, 91 insertions(+), 250 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 6616d1f67..45dedc847 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -35,266 +35,111 @@ pipeline { stages { - stage('Check Out') { - parallel { - // - // Run the prepare stage on the stardog node, check out all the repos there - // - // NOTE: For now it' the stardog node since we have more processing power there and there are things - // installed there that we don't have on the jenkins master (such as "nix.sh") - // - stage('Stardog Node') { - - agent { - label 'stardog' - } - - steps { - - //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'PENDING' - - // - // Clean the workspace on this node, for now. Takes more time but we need to test this with a clean slate - // every time. Once this all works fine we can skip this step. - // - sh "rm -rf ${env.WORKSPACE}/*" - - // - // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script) - // - step([$class: 'CopyArtifact', filter: '**/rdf-toolkit.jar', fingerprintArtifacts: true, flatten: true, projectName: 'rdf-toolkit-build']) - // - // Copy the pellet jar from the build-pellet job - // - step([$class: 'CopyArtifact', filter: '**/pellet-cli-*.jar', fingerprintArtifacts: true, flatten: true, projectName: 'build-pellet', target: 'pellet']) - // - // Check out the fibo repo' content into the ./fibo directory - // - dir('fibo') { - checkout scm - echo 'Checked out fibo repo' - } - // - // Then check out the fibo-infra repo content into the ./fibo-infra directory - // - dir('fibo-infra') { - // - // If you want to check out a specific version of fibo-infra add the "branch:" parameter - // - //git branch: 'INFRA-161', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' - git branch: 'INFRA-224', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' - // - // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' - echo 'Checked out fibo-infra repo' - // - // Stash the nomagic templates so that they can be unstashed on the nomagix box - // - stash includes: 'magicdraw/Concept Modeler/**', name: 'nomagic-templates' - // - // Stash all the tools in the bin directory - // - stash includes: 'bin/**', name: 'infra-bin' - // - // Stash all the jenkins related stuff in the jenkins directory - // - stash includes: 'jenkins/**', name: 'infra-jenkins' - } - // - // Then run the init command on the fibo-publish script which will set all the GIT_* and JIRA_* vars - // correctly and store their values in some .env files. - // - dir('fibo-infra') { - echo 'Execute the publish-fibo.sh script:' - sh './jenkins/bin/publish-fibo.sh init' - echo 'Finished executing the publish-fibo script' - } - initEnvironment() - // - // Now stash all files in the env directory for reuse in other workspaces on other agents/slaves - // - stash includes: 'env/**', name: 'environment' - // - // Then check out the LCC repo into the ./LCC directory - // - dir('LCC') { - git url: 'https://github.com/edmcouncil/LCC.git', credentialsId: 'edmcjenkins' - echo 'Checked out the LCC repo' - } - echo 'Checked it all out' - } - } // end of stage 'Prepare Stardog Node' - - stage('NoMagic Node') { + stage('Prepare') { + agent { + label 'stardog' + } - agent { - label 'nomagic' - } + steps { - steps { - // - // Clean the workspace on this node - // - sh "rm -rf ${env.WORKSPACE}/*" + //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'PENDING' - // - // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script) - // - step([$class: 'CopyArtifact', filter: '**/rdf-toolkit.jar', fingerprintArtifacts: true, flatten: true, projectName: 'rdf-toolkit-build']) + // + // Clean the workspace on this node, for now. Takes more time but we need to test this with a clean slate + // every time. Once this all works fine we can skip this step. + // + // JG>We can clean the workspace in the settings of the job here: https://jenkins.edmcouncil.org/job/fibo/configure + // (Look under Behaviors). That' better than to always do it since this causes a lot of extra time. + //sh "rm -rf ${env.WORKSPACE}/*" - dir('fibo') { - checkout scm - echo 'Checked out fibo repo' - } - } - } // end of stage 'Prepare NoMagic Node' - } // end of parallel + // + // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script) + // + step([$class: 'CopyArtifact', filter: '**/rdf-toolkit.jar', fingerprintArtifacts: true, flatten: true, projectName: 'rdf-toolkit-build']) + // + // Copy the pellet jar from the build-pellet job + // + step([$class: 'CopyArtifact', filter: '**/pellet-cli-*.jar', fingerprintArtifacts: true, flatten: true, projectName: 'build-pellet', target: 'pellet']) + // + // Check out the fibo repo' content into the ./fibo directory + // + dir('fibo') { + checkout scm + echo 'Checked out fibo repo' + } + // + // Then check out the fibo-infra repo content into the ./fibo-infra directory + // + dir('fibo-infra') { + // + // If you want to check out a specific version of fibo-infra add the "branch:" parameter + // + git branch: 'INFRA-224', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + // + // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + echo 'Checked out fibo-infra repo' + // + // Stash all the tools in the bin directory + // + stash includes: 'bin/**', name: 'infra-bin' + // + // Stash all the jenkins related stuff in the jenkins directory + // + stash includes: 'jenkins/**', name: 'infra-jenkins' + } + // + // Then run the init command on the fibo-publish script which will set all the GIT_* and JIRA_* vars + // correctly and store their values in some .env files. + // + dir('fibo-infra') { + echo 'Execute the publish-fibo.sh script:' + sh './jenkins/bin/publish-fibo.sh init' + echo 'Finished executing the publish-fibo script' + } + initEnvironment() + // + // Now stash all files in the env directory for reuse in other workspaces on other agents/slaves + // + stash includes: 'env/**', name: 'environment' + // + // Then check out the LCC repo into the ./LCC directory + // + dir('LCC') { + git url: 'https://github.com/edmcouncil/LCC.git', credentialsId: 'edmcjenkins' + echo 'Checked out the LCC repo' + } + echo 'Checked it all out' + } } // end of stage 'Prepare' stage('Build') { - parallel { + agent { + label 'stardog' + } + steps { // - // Run on the stardog node + // Execute the publish script // - // NOTE: For now it' the stardog node since we have more processing power there and there are things - // installed there that we don't have on the jenkins master (such as "nix.sh") + // TODO: Split this script up in separate "stages" in this repository // - stage('Stardog Node') { - - agent { - label 'stardog' - } - - steps { - // - // Execute the publish script - // - // TODO: Split this script up in separate "stages" in this repository - // - dir('fibo-infra') { - echo "Execute the publish-fibo.sh script:" - sh "./jenkins/bin/publish-fibo.sh" - } - - // - // Archive the artifacts generated by the publish-fibo.sh script - // - dir('target') { - stash([ - name: 'publish-script-output-stardog-node', - includes: '**', - excludes: '**/.git, **/.gitignore', - useDefaultExcludes: true - ]) - } - } - } // end of stage 'Build on Stardog Node' + dir('fibo-infra') { + echo "Execute the publish-fibo.sh script:" + sh "./jenkins/bin/publish-fibo.sh" + } // - // Build the artifacts that that need the nomagic installation + // Archive the artifacts generated by the publish-fibo.sh script // - stage('NoMagic Node') { - - agent { - label 'nomagic' - } - environment { - NOMAGIC_CREDS = credentials('50cac519-d41c-4765-8563-c43b7f55c877') - NOMAGIC_USERID = "${env.NOMAGIC_CREDS_USR}" - NOMAGIC_PASSWD = "${env.NOMAGIC_CREDS_PSW}" - } - - steps { - unstash 'environment' - unstash 'nomagic-templates' - unstash 'infra-bin' - unstash 'infra-jenkins' - - initEnvironment() - - echo "NOMAGIC_USERID=${NOMAGIC_USERID}" - - // - // Copy the nomagic templates (as they are stored in the fibo-infra repo) - // to the right location - // - dir('magicdraw/Concept Modeler') { - sh 'pwd' - sh 'find .' - sh 'cp -vr . /home/ec2-user/MagicDraw/data/defaults/data/reports/Concept\\ Modeler/' - } - - /* JG>Disabling this now until nomagic process actually gets its input from this job - - withCredentials([ - usernamePassword( - credentialsId: '50cac519-d41c-4765-8563-c43b7f55c877', - passwordVariable: 'NOMAGIC_USERID', - usernameVariable: 'NOMAGIC_PASSWD' - ) - ]) { - echo "inside withCredentials block: NOMAGIC_USERID=${NOMAGIC_USERID}" - withEnv([ - "NOMAGIC_USERID=${env.NOMAGIC_USERID}", - "NOMAGIC_PASSWD=${env.NOMAGIC_PASSWD}", - "NOMAGIC_SERVER=twc184.nomagic.com:3579" - ]) { - - echo "inside withEnv() block:" - sh 'echo NOMAGIC_USERID=\${NOMAGIC_USERID}' - sh 'echo NOMAGIC_SERVER=\${NOMAGIC_SERVER}' - // - // Execute the publish script - // - // TODO: Split this script up in separate "stages" in this repository - // - echo "Execute the publish-fibo.sh script:" - sh "./jenkins/bin/publish-fibo.sh" - } - } - */ - - // - // Instead, we're just dumping the artifacts produced by the following job here and - // process it further just as if it has been generated here: - // https://jenkins.edmcouncil.org/job/NoMagic/job/fibo-publish-nomagic/ - // - step([ - $class: 'CopyArtifact', - excludes: '**/*.jar, **/*.csv, **/PRODIN.html, **/DEVIN.html', - filter: '**', - projectName: 'NoMagic/fibo-publish-nomagic', - target: "target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}" - ]) - echo "Copied nomagic output to target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}:" - dir("target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}") { - sh "mv -v PRODUCTION.html production.html || true" - sh "mv -v DEVELOPMENT.html development.html || true" - } - sh "find target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}" - step([ - $class: 'CopyArtifact', - excludes: '*.jar, *.html, *.css, *.js', - filter: '*.csv', - projectName: 'NoMagic/fibo-publish-nomagic', - target: "target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}" - ]) - echo "Copied nomagic output to target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}:" - dir("target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}") { - sh "mv -v datadictionaryPROD.csv production.csv || true" - sh "mv -v datadictionaryDEV.csv development.csv || true" - } - sh "find target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}" - - // - // Archive the artifacts generated by the publish-fibo.sh script - // - dir('target') { - stash includes: '**', name: 'publish-script-output-nomagic-node' - } - } - } // end of stage 'Build on NoMagic Node' - } // end of parallel + dir('target') { + stash([ + name: 'publish-script-output-stardog-node', + includes: '**', + excludes: '**/.git, **/.gitignore', + useDefaultExcludes: true + ]) + } + } } // end of stage 'Build' // @@ -326,10 +171,6 @@ pipeline { sh 'find . -name \'.git\' -exec rm -rf {} \\; || true' sh 'find . -name \'.gitignore\' -exec rm -f {} \\; || true' - echo 'Unstashing the output of the publish-script as it ran on the nomagic node' - unstash 'publish-script-output-nomagic-node' - sh 'ls -al' - echo "Copy all generated content to ${env.NGINX_SPEC_ROOT}:" sh 'ls -al ${NGINX_SPEC_ROOT}/' sh 'cp -vr . ${NGINX_SPEC_ROOT}/' From 037e0a20477ce91b8761cb7bbfe9419ddd8d986d Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 18:04:00 +0000 Subject: [PATCH 05/15] INFRA-225 Using INFRA-225 branch in fibo-infra --- etc/process/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 45dedc847..057f4e571 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -74,7 +74,7 @@ pipeline { // // If you want to check out a specific version of fibo-infra add the "branch:" parameter // - git branch: 'INFRA-224', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + git branch: 'INFRA-225', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' // // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' echo 'Checked out fibo-infra repo' From fdecdbb37b264f2496ca738fe1f1155fdafb8a16 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 18:13:41 +0000 Subject: [PATCH 06/15] INFRA-225 Added NodeJS to the Jenkinsfile --- etc/process/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 057f4e571..eb121fbfe 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -116,6 +116,13 @@ pipeline { agent { label 'stardog' } + tools { + // + // This installs the NodeJS tools as they are configured in the "Global Tool Configuration" + // of Jenkins: https://jenkins.edmcouncil.org/configureTools/ + // + nodejs 'NodeJS' + } steps { // From a2edc3e14e254255c5ade4303643732a3293668f Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 19:53:09 +0000 Subject: [PATCH 07/15] INFRA-225 Suppressing output of final copy command --- etc/process/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index eb121fbfe..c3353e15f 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -180,7 +180,9 @@ pipeline { echo "Copy all generated content to ${env.NGINX_SPEC_ROOT}:" sh 'ls -al ${NGINX_SPEC_ROOT}/' - sh 'cp -vr . ${NGINX_SPEC_ROOT}/' + sh 'cp -vr . ${NGINX_SPEC_ROOT}/ > published-files.log 2>&1' + echo 'Successfully copied all files to destination, see published-files.log in the workspace' + archiveArtifacts artifacts: 'published-files.log', fingerprint: true //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'SUCCESS' } From 58781498e1453b9cfd6c268088d789d1b4f4f507 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 20:20:18 +0000 Subject: [PATCH 08/15] INFRA-225 Now also archiving *.tar.gz.log files --- etc/process/Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index c3353e15f..bea30d1cd 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -183,6 +183,10 @@ pipeline { sh 'cp -vr . ${NGINX_SPEC_ROOT}/ > published-files.log 2>&1' echo 'Successfully copied all files to destination, see published-files.log in the workspace' archiveArtifacts artifacts: 'published-files.log', fingerprint: true + sh 'rm published-files.log' + + archiveArtifacts artifacts: '**.tar.gz.log', fingerprint: true + sh "find . -name '*.tar.gz.log' -delete" //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'SUCCESS' } From 6fa55b3367abc9af54828bd198e2282a20666673 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Thu, 28 Dec 2017 20:38:43 +0000 Subject: [PATCH 09/15] INFRA-225 Now archiving all .log files --- etc/process/Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index bea30d1cd..5487bd4d6 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -177,16 +177,15 @@ pipeline { sh 'ls -al' sh 'find . -name \'.git\' -exec rm -rf {} \\; || true' sh 'find . -name \'.gitignore\' -exec rm -f {} \\; || true' + sh 'find . -name \'temp*\' -exec rm -f {} \\; || true' echo "Copy all generated content to ${env.NGINX_SPEC_ROOT}:" sh 'ls -al ${NGINX_SPEC_ROOT}/' sh 'cp -vr . ${NGINX_SPEC_ROOT}/ > published-files.log 2>&1' echo 'Successfully copied all files to destination, see published-files.log in the workspace' - archiveArtifacts artifacts: 'published-files.log', fingerprint: true - sh 'rm published-files.log' - archiveArtifacts artifacts: '**.tar.gz.log', fingerprint: true - sh "find . -name '*.tar.gz.log' -delete" + archiveArtifacts artifacts: '**/*.log', fingerprint: true + sh 'find . -name \'*.log\' -exec rm -f {} \\; || true' //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'SUCCESS' } From 49e4eabc7816e383a191ba0dd4b6c13aa3f0e94e Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Fri, 29 Dec 2017 15:55:36 +0000 Subject: [PATCH 10/15] INFRA-225 Added some cleanups --- etc/process/Jenkinsfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 5487bd4d6..8ef357a70 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -174,10 +174,16 @@ pipeline { echo 'Unstashing the output of the publish-script as it ran on the stardog node' unstash 'publish-script-output-stardog-node' - sh 'ls -al' + // + // TODO: The cleanups below should be done by the publish script + // sh 'find . -name \'.git\' -exec rm -rf {} \\; || true' sh 'find . -name \'.gitignore\' -exec rm -f {} \\; || true' - sh 'find . -name \'temp*\' -exec rm -f {} \\; || true' + sh 'find . -type d -name node_modules -exec rm -rf {} \\; || true' + sh 'find . -type d -name build -exec rm -rf {} \\; || true' + sh 'find . -type d -name \'temp*\' -exec rm -rf {} \\; || true' + sh 'find . -type f -name \'temp*\' -exec rm -f {} \\; || true' + sh 'ls -al' echo "Copy all generated content to ${env.NGINX_SPEC_ROOT}:" sh 'ls -al ${NGINX_SPEC_ROOT}/' From 9873f088b61318508033453649c2f3c7b323d04d Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Fri, 29 Dec 2017 16:31:15 +0000 Subject: [PATCH 11/15] INFRA-225 Trying to get linux tools to emit UTF8 output --- etc/process/Jenkinsfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 8ef357a70..a4c6e7f22 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -10,6 +10,13 @@ pipeline { agent none + environment { + FIBO_INFRA_BRANCH = 'INFRA-225' + LC_ALL = 'en_US.UTF-8' + LANG = 'en_US.UTF-8' + LANGUAGE = 'en_US.UTF-8' + } + options { buildDiscarder( @@ -74,7 +81,7 @@ pipeline { // // If you want to check out a specific version of fibo-infra add the "branch:" parameter // - git branch: 'INFRA-225', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' + git branch: env.FIBO_INFRA_BRANCH, credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' // // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' echo 'Checked out fibo-infra repo' From a42722d412845babb049d1eabe7d79480a633ab1 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Fri, 29 Dec 2017 16:47:44 +0000 Subject: [PATCH 12/15] INFRA-225 Added cleanWs() --- etc/process/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index a4c6e7f22..3b9b9ebb4 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -132,6 +132,7 @@ pipeline { } steps { + cleanWs() // // Execute the publish script // From d420cb52e1f77d90362070077aac2e7f0c8dc6ec Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Fri, 29 Dec 2017 18:40:48 +0000 Subject: [PATCH 13/15] INFRA-225 Yet another way to clean . . . x --- etc/process/Jenkinsfile | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 3b9b9ebb4..947091017 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -46,18 +46,16 @@ pipeline { agent { label 'stardog' } + environment { + LC_ALL = 'en_US.UTF-8' + LANG = 'en_US.UTF-8' + LANGUAGE = 'en_US.UTF-8' + } steps { - - //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'PENDING' - - // - // Clean the workspace on this node, for now. Takes more time but we need to test this with a clean slate - // every time. Once this all works fine we can skip this step. - // - // JG>We can clean the workspace in the settings of the job here: https://jenkins.edmcouncil.org/job/fibo/configure - // (Look under Behaviors). That' better than to always do it since this causes a lot of extra time. - //sh "rm -rf ${env.WORKSPACE}/*" + echo "Cleaning workspace:" + sh 'rm -rf ${WORKSPACE}/* || true' + echo "Done cleaning" // // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script) @@ -84,24 +82,24 @@ pipeline { git branch: env.FIBO_INFRA_BRANCH, credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' // // git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git' - echo 'Checked out fibo-infra repo' + echo "Checked out fibo-infra repo, branch ${env.FIBO_INFRA_BRANCH}" // // Stash all the tools in the bin directory // - stash includes: 'bin/**', name: 'infra-bin' + //stash includes: 'bin/**', name: 'infra-bin' // // Stash all the jenkins related stuff in the jenkins directory // - stash includes: 'jenkins/**', name: 'infra-jenkins' + //stash includes: 'jenkins/**', name: 'infra-jenkins' } // // Then run the init command on the fibo-publish script which will set all the GIT_* and JIRA_* vars // correctly and store their values in some .env files. // dir('fibo-infra') { - echo 'Execute the publish-fibo.sh script:' + echo 'Execute the publish-fibo.sh script (just to initialize the environment variables):' sh './jenkins/bin/publish-fibo.sh init' - echo 'Finished executing the publish-fibo script' + echo 'Finished executing the publish-fibo script (init section)' } initEnvironment() // @@ -132,7 +130,6 @@ pipeline { } steps { - cleanWs() // // Execute the publish script // @@ -175,10 +172,12 @@ pipeline { steps { + echo "Cleaning workspace:" + sh 'rm -rf .* * || true' + echo "Done cleaning" + sh 'test -d ${NGINX_SPEC_ROOT}' sh 'pwd' - echo "Cleaning workspace before unstashing fresh content" - sh 'rm -rf *' echo 'Unstashing the output of the publish-script as it ran on the stardog node' unstash 'publish-script-output-stardog-node' @@ -219,6 +218,11 @@ pipeline { def initEnvironment() { + // + // Read the GIT and JIRA environment variables from files in the ./env directory + // that have been created by the publish-fibo.sh script so that we can reuse that + // logic elsewhere in the Jenkinsfile pipeline. + // env.GIT_BRANCH = readFile './env/GIT_BRANCH'.trim() env.GIT_TAG_NAME = readFile './env/GIT_TAG_NAME'.trim() env.GIT_AUTHOR = readFile './env/GIT_AUTHOR'.trim() From 08266737da968e0261ba20d11a6727f68db4b7c9 Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Tue, 2 Jan 2018 17:28:36 +0000 Subject: [PATCH 14/15] INFRA-225 Parallelized the build --- etc/process/Jenkinsfile | 81 ++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index 947091017..f128c68c0 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -117,28 +117,83 @@ pipeline { } } // end of stage 'Prepare' - stage('Build') { + stage('Build Product Ontology') { agent { label 'stardog' } - tools { - // - // This installs the NodeJS tools as they are configured in the "Global Tool Configuration" - // of Jenkins: https://jenkins.edmcouncil.org/configureTools/ - // - nodejs 'NodeJS' - } steps { // // Execute the publish script // - // TODO: Split this script up in separate "stages" in this repository + sh "./fibo-infra/jenkins/bin/publish-fibo.sh ontology" // - dir('fibo-infra') { - echo "Execute the publish-fibo.sh script:" - sh "./jenkins/bin/publish-fibo.sh" + // Now stash all files in the env directory for reuse in the other worker nodes + // used by the next products + // + stash includes: '**', name: 'all-ontology-data' + } + } // end of stage 'Build Product Ontology' + + stage('Build Other Products') { + parallel { + stage("Build Widoco") { + agent { + label 'stardog' + } + steps { + unstash 'all-ontology-data' + sh "./fibo-infra/jenkins/bin/publish-fibo.sh widoco" + } + } + stage("Build Glossary") { + agent { + label 'stardog' + } + tools { + // + // This installs the NodeJS tools as they are configured in the "Global Tool Configuration" + // of Jenkins: https://jenkins.edmcouncil.org/configureTools/ + // + nodejs 'NodeJS' + } + steps { + unstash 'all-ontology-data' + sh "./fibo-infra/jenkins/bin/publish-fibo.sh glossary" + } + } + stage("Build Vocabulary") { + agent { + label 'stardog' + } + steps { + unstash 'all-ontology-data' + sh "./fibo-infra/jenkins/bin/publish-fibo.sh vocabulary" + } + } + stage("Build Datadictionary") { + agent { + label 'stardog' + } + steps { + unstash 'all-ontology-data' + sh "./fibo-infra/jenkins/bin/publish-fibo.sh datadictionary" + } } + } + } // end of stage 'Build Other Products' + + stage('Build Final Content') { + agent { + label 'stardog' + } + + steps { + // + // Now after all the above is done, make sure we run the final + // publish step which zips it all up into the target directory + // + sh "./fibo-infra/jenkins/bin/publish-fibo.sh publish" // // Archive the artifacts generated by the publish-fibo.sh script @@ -152,7 +207,7 @@ pipeline { ]) } } - } // end of stage 'Build' + } // end of stage 'Build Final Content' // // Run the publish on the master jenkins agent by just copying all the generated artifacts right into the workspace From 2d8364fe22d96b93ad7d299645a5b460fc410d6a Mon Sep 17 00:00:00 2001 From: Jacobus Geluk Date: Tue, 2 Jan 2018 19:58:30 +0000 Subject: [PATCH 15/15] INFRA-225 FIBO_INFRA_BRANCH=master --- etc/process/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/process/Jenkinsfile b/etc/process/Jenkinsfile index f128c68c0..b45a14d39 100644 --- a/etc/process/Jenkinsfile +++ b/etc/process/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { agent none environment { - FIBO_INFRA_BRANCH = 'INFRA-225' + FIBO_INFRA_BRANCH = 'master' LC_ALL = 'en_US.UTF-8' LANG = 'en_US.UTF-8' LANGUAGE = 'en_US.UTF-8'