From cde1fe05cd23738dec6e2b7519d8f13704f8794a Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Sun, 27 Sep 2020 18:08:37 +0200
Subject: [PATCH 01/22] first attempt at mock el8

---
 rhel-8-script-env-vars-puppet-only.groovy |  9 +++++++++
 rhel-8-script-env-vars-rpm.groovy         | 12 ++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 rhel-8-script-env-vars-puppet-only.groovy
 create mode 100644 rhel-8-script-env-vars-rpm.groovy

diff --git a/rhel-8-script-env-vars-puppet-only.groovy b/rhel-8-script-env-vars-puppet-only.groovy
new file mode 100644
index 0000000..92e3253
--- /dev/null
+++ b/rhel-8-script-env-vars-puppet-only.groovy
@@ -0,0 +1,9 @@
+env.REPO_ID="3825"
+env.PUPPET_REPO_ID="9"
+env.TESTVM_HOSTCOLLECTION="hc-soe-el8-test"
+env.PUPPET_REPO="/var/www/html/pub/soe-puppet-only"
+env.CV="cv-puppet-only"
+env.CV_PASSIVE_LIST=""
+env.CCV_NAME_PATTERN=""
+env.CONDITIONAL_VM_BUILD=false
+env.PUPPET_DONE_SLEEP="0"
diff --git a/rhel-8-script-env-vars-rpm.groovy b/rhel-8-script-env-vars-rpm.groovy
new file mode 100644
index 0000000..8f847ec
--- /dev/null
+++ b/rhel-8-script-env-vars-rpm.groovy
@@ -0,0 +1,12 @@
+//this is for RHEL8 only as we build packages and shove them in a RHEL8 only yum repo
+env.REPO_ID="3825"
+env.PUPPET_REPO_ID="8"
+env.TESTVM_HOSTCOLLECTION="hc-soe-el8-test"
+env.YUM_REPO="/var/www/html/pub/soe-repo/rhel8"
+env.PUPPET_REPO="/var/www/html/pub/soe-puppet"
+env.CV="cv-soe-ci-el8"
+env.CV_PASSIVE_LIST=""
+env.CCV_NAME_PATTERN=""
+env.CONDITIONAL_VM_BUILD=false
+env.MOCK_CONFIG="rhel-8-x86_64"
+env.PUPPET_DONE_SLEEP="75"

From 1bced18beba0e7b004a97c1b6b396de637dc2c83 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Sun, 27 Sep 2020 18:52:19 +0200
Subject: [PATCH 02/22] temporarily disable puppet pushing

---
 puppetpush.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/puppetpush.sh b/puppetpush.sh
index 2972f92..5758de9 100755
--- a/puppetpush.sh
+++ b/puppetpush.sh
@@ -8,6 +8,14 @@
 # Load common parameter variables
 . $(dirname "${0}")/common.sh
 
+### disable for a quick test
+### pcfe, 2020-08-27
+
+inform "temporarily disabled"
+exit 0
+
+###
+
 if [[ -z ${PUSH_USER} ]] || [[ -z ${SATELLITE} ]]
 then
     err "PUSH_USER or SATELLITE not set or not found"

From 0112ccbb48e93c87102f5092e1bb8193ec10a5c0 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 18:56:23 +0200
Subject: [PATCH 03/22] WIP, after test, install a golden VM

golden simply because I did not come up
with a better name
---
 buildgoldenvms.sh      | 123 +++++++++++++++++++++++++++++++++++++++++
 common.sh              |  25 +++++++++
 script-env-vars.groovy |   1 +
 3 files changed, 149 insertions(+)
 create mode 100755 buildgoldenvms.sh

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
new file mode 100755
index 0000000..6ed87f5
--- /dev/null
+++ b/buildgoldenvms.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# Instruct Foreman to rebuild the VMs you wan to image (as in these will make your golden image)
+#
+# e.g ${WORKSPACE}/scripts/buildgoldenvms.sh 'test'
+#
+# this will tell Foreman to rebuild all machines in host collection GOLDENVM_HOSTCOLLECTION
+
+# Load common parameter variables
+. $(dirname "${0}")/common.sh
+
+if [[ -z ${PUSH_USER} ]] || [[ -z ${SATELLITE} ]]  || [[ -z ${RSA_ID} ]] \
+   || [[ -z ${ORG} ]] || [[ -z ${GOLDENVM_HOSTCOLLECTION} ]]
+then
+    err "Environment variable PUSH_USER, SATELLITE, RSA_ID, ORG " \
+        "or GOLDENVM_HOSTCOLLECTION not set or not found."
+    exit ${WORKSPACE_ERR}
+fi
+
+get_goden_vm_list # populate GOLDEN_VM_LIST
+
+# TODO: Error out if no test VM's are available.
+if [ $(echo ${#GOLDEN_VM_LIST[@]}) -eq 0 ]; then
+  err "No test VMs configured in Satellite"
+fi
+
+# rebuild test VMs
+for I in "${GOLDEN_VM_LIST[@]}"
+do
+    inform "Rebuilding VM ID $I"
+    ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+        "hammer host update --id $I --build yes"
+
+    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --id $I" | grep Power | cut -f2 -d: | tr -d ' ')
+
+    # different hypervisors report power status with different words. parse and get a single word per status
+    # KVM uses running / shutoff
+    # VMware uses poweredOn / poweredOff
+    # libvirt uses running / off
+    # add other hypervisors as you come across them and please submit to https://github.com/RedHatEMEA/soe-ci
+
+    case "${_PROBED_STATUS}" in
+      running)
+        _STATUS=On
+        ;;
+      poweredOn)
+        _STATUS=On
+        ;;
+      up)
+        _STATUS=On
+        ;;
+      shutoff)
+        _STATUS=Off
+        ;;
+      poweredOff)
+        _STATUS=Off
+        ;;
+      down)
+        _STATUS=Off
+        ;;
+      off)
+        _STATUS=Off
+        ;;
+      *)
+        echo "can not parse power status, please review $0"
+    esac
+
+    if [[ ${_STATUS} == 'On' ]]
+    then
+        # forcefully poweroff the SUT
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host stop --force --id $I"
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host start --id $I"
+    elif [[ ${_STATUS} == 'Off' ]]
+    then
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host start --id $I"
+    else
+        err "Host $I is neither running nor shutoff. No action possible!"
+        exit 1
+    fi
+done
+
+
+# we need to wait until all the test machines have been rebuilt by foreman
+# this check was previously only in pushtests, but when using pipelines 
+# it's more sensible to wait here while the machines are in build mode
+# the ping and ssh checks must remain in pushtests.sh
+# as a pupet only build will not call this script
+
+declare -A vmcopy # declare an associative array to copy our VM array into
+for I in "${GOLDEN_VM_LIST[@]}"; do vmcopy[$I]=$I; done
+
+WAIT=0
+while [[ ${#vmcopy[@]} -gt 0 ]]
+do
+    inform "Waiting 1 minute"
+    sleep 60
+    ((WAIT+=60))
+    for I in "${vmcopy[@]}"
+    do
+        inform "Checking if host $I is in build mode."
+        status=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host info --name $I | \
+            grep -e \"Managed.*yes\" -e \"Enabled.*yes\" -e \"Build.*no\" \
+                | wc -l")
+        # Check if status is OK, then the SUT will have left build mode
+        if [[ ${status} == 3 ]]
+        then
+            tell "host $I no longer in build mode."
+            unset vmcopy[$I]
+        else
+            tell "host $I is still in build mode."
+        fi
+    done
+    if [[ ${WAIT} -gt 6000 ]]
+    then
+        err "At least one host still in build mode after 6000 seconds. Exiting."
+        exit 1
+    fi
+done
+
diff --git a/common.sh b/common.sh
index a2286da..4cc5231 100755
--- a/common.sh
+++ b/common.sh
@@ -64,3 +64,28 @@ function get_test_vm_list() {
 		fi
 	done
 }
+
+# and the same again, but for GOLDEN_VM_LIST
+function get_golden_vm_list() {
+	local K=0
+	for I in $(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+		"hammer host-collection hosts --organization \"${ORG}\" \
+			--name \"$GOLDENVM_HOSTCOLLECTION\" \
+		| tail -n +4 | cut -f2 -d \"|\" | head -n -1")
+	do
+		# If CONDITIONAL_VM_BUILD is 'true', only keep VMs commented
+		# with modified #content# as listed in $MODIFIED_CONTENT_FILE
+		# If the file is empty or doesn't exist, we test everything
+		# as it hints at a script change.
+		if [[ "${CONDITIONAL_VM_BUILD}" != 'true' ]] || \
+			[[ ! -s "${MODIFIED_CONTENT_FILE}" ]] || \
+			ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+			"hammer --output yaml host info --name \"${I}\"" \
+				| grep "^Comment:" \
+				| grep -Fqf "${MODIFIED_CONTENT_FILE}"
+		then
+			GOLDEN_VM_LIST[$K]=$I
+			((K+=1))
+		fi
+	done
+}
diff --git a/script-env-vars.groovy b/script-env-vars.groovy
index 40fe606..aab35e5 100644
--- a/script-env-vars.groovy
+++ b/script-env-vars.groovy
@@ -6,4 +6,5 @@ env.KNOWN_HOSTS="/var/lib/jenkins/.ssh/known_hosts"
 env.SATELLITE="satellite.internal.pcfe.net"
 env.ORG="Sat Test"
 env.TESTVM_ENV="2"
+env.GOLDENVM_ENV="14"
 env.TEST_ROOT="redhat geheim"

From 1138be5092394249ae12f9ffc2a3c49bf670ece8 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 19:03:05 +0200
Subject: [PATCH 04/22] WIP, ensure golden VMs are shut down cleanly

---
 shutdowngoldenvms.sh | 81 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100755 shutdowngoldenvms.sh

diff --git a/shutdowngoldenvms.sh b/shutdowngoldenvms.sh
new file mode 100755
index 0000000..2edc2d6
--- /dev/null
+++ b/shutdowngoldenvms.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# power off the golden VMs
+# hammer (without the --force flag) will attempt a clean shutdown
+#
+# e.g ${WORKSPACE}/scripts/poweroffgoldenvms.sh 'test'
+#
+
+# Load common parameter variables
+. $(dirname "${0}")/common.sh
+
+if [[ -z ${PUSH_USER} ]] || [[ -z ${SATELLITE} ]]  || [[ -z ${RSA_ID} ]] \
+   || [[ -z ${ORG} ]] || [[ -z ${GOLDENVM_HOSTCOLLECTION} ]]
+then
+    err "Environment variable PUSH_USER, SATELLITE, RSA_ID, ORG " \
+        "or GOLDENVM_HOSTCOLLECTION not set or not found."
+    exit ${WORKSPACE_ERR}
+fi
+
+get_golden_vm_list # populate GOLDEN_VM_LIST
+
+if [ $(echo ${#GOLDEN_VM_LIST[@]}) -eq 0 ]; then
+  err "No golden VMs configured in Satellite"
+  exit 1
+fi
+
+# shutdown golden VMs
+for I in "${GOLDEN_VM_LIST[@]}"
+do
+    inform "Checking status of VM ID $I"
+
+    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --id $I" | grep Power | cut -f2 -d: | tr -d ' ')
+
+    # different hypervisors report power status with different words. parse and get a single word per status
+    # KVM uses running / shutoff
+    # VMware uses poweredOn / poweredOff
+    # libvirt uses running / off
+    # add other hypervisors as you come across them and please submit to https://github.com/RedHatEMEA/soe-ci
+
+    case "${_PROBED_STATUS}" in
+      running)
+        _STATUS=On
+        ;;
+      poweredOn)
+        _STATUS=On
+        ;;
+      up)
+        _STATUS=On
+        ;;
+      shutoff)
+        _STATUS=Off
+        ;;
+      poweredOff)
+        _STATUS=Off
+        ;;
+      down)
+        _STATUS=Off
+        ;;
+      off)
+        _STATUS=Off
+        ;;
+      *)
+        echo "can not parse power status, please review $0"
+    esac
+
+    # n.b. kickstart can either reboot or power down at the end, so we must handle both cases
+    # also, since we do not pushtests.sh aginst the golden VMs,
+    # ensure it's up before attempting clean shutdown
+    if [[ ${_STATUS} == 'On' ]]
+    then
+        inform "Shutting down VM ID $I"
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host stop --id $I"
+    elif [[ ${_STATUS} == 'Off' ]]
+    then
+        inform "VM ID $I seems off already, no action done."
+    else
+        err "Host $I is neither running nor shutoff. No action possible!"
+        exit 1
+    fi
+done

From 0698dcc86f52b99f41a8434bb70af628a09b1be8 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 19:24:15 +0200
Subject: [PATCH 05/22] WIP, start golden VMs

---
 buildgoldenvms.sh |  2 +-
 startgoldenvms.sh | 77 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100755 startgoldenvms.sh

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
index 6ed87f5..838ee79 100755
--- a/buildgoldenvms.sh
+++ b/buildgoldenvms.sh
@@ -17,7 +17,7 @@ then
     exit ${WORKSPACE_ERR}
 fi
 
-get_goden_vm_list # populate GOLDEN_VM_LIST
+get_golden_vm_list # populate GOLDEN_VM_LIST
 
 # TODO: Error out if no test VM's are available.
 if [ $(echo ${#GOLDEN_VM_LIST[@]}) -eq 0 ]; then
diff --git a/startgoldenvms.sh b/startgoldenvms.sh
new file mode 100755
index 0000000..3872f39
--- /dev/null
+++ b/startgoldenvms.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Instruct Foreman to start the golden VMs (just in case they are off)
+#
+# e.g ${WORKSPACE}/scripts/startgoldenvms.sh 'test'
+#
+# this will tell Foreman to start all machines in hostgroup GOLDENVM_HOSTGROUP
+
+# Load common parameter variables
+. $(dirname "${0}")/common.sh
+
+if [[ -z ${PUSH_USER} ]] || [[ -z ${SATELLITE} ]]  || [[ -z ${RSA_ID} ]] \
+   || [[ -z ${ORG} ]] || [[ -z ${GOLDENVM_HOSTCOLLECTION} ]]
+then
+    err "Environment variable PUSH_USER, SATELLITE, RSA_ID, ORG " \
+        "or GOLDENVM_HOSTCOLLECTION not set or not found."
+    exit ${WORKSPACE_ERR}
+fi
+
+get_golden_vm_list # populate GOLDEN_VM_LIST
+
+if [ $(echo ${#GOLDEN_VM_LIST[@]}) -eq 0 ]; then
+  err "No golden VMs configured in Satellite"
+  exit 1
+fi
+
+# rebuild golden VMs
+for I in "${GOLDEN_VM_LIST[@]}"
+do
+    inform "Making sure VM ID $I is on"
+
+    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --id $I" | grep Power | cut -f2 -d: | tr -d ' ')
+
+    # different hypervisors report power status with different words. parse and get a single word per status
+    # KVM uses running / shutoff
+    # VMware uses poweredOn / poweredOff
+    # add other hypervisors as you come across them and please submit to https://github.com/RedHatEMEA/soe-ci
+
+    case "${_PROBED_STATUS}" in
+      running)
+        _STATUS=On
+        ;;
+      poweredOn)
+        _STATUS=On
+        ;;
+      up)
+        _STATUS=On
+        ;;
+      shutoff)
+        _STATUS=Off
+        ;;
+      poweredOff)
+        _STATUS=Off
+        ;;
+      down)
+        _STATUS=Off
+        ;;
+      off)
+        _STATUS=Off
+        ;;
+      *)
+        echo "can not parse power status, please review $0"
+    esac
+
+    if [[ ${_STATUS} == 'On' ]]
+    then
+        inform "Host $I is already on."
+    elif [[ ${_STATUS} == 'Off' ]]
+    then
+        inform "Host $I is already off, switching it on."
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host start --id $I"
+    else
+        err "Host $I is neither running nor shutoff. No action possible!"
+        exit 1
+    fi
+done

From 478eb46f7e372ee42a857d9b0b49ebfaa0ee53f1 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 19:25:30 +0200
Subject: [PATCH 06/22] fix a forgotten todo

---
 powerofftestvms.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/powerofftestvms.sh b/powerofftestvms.sh
index 948d39e..a85b701 100755
--- a/powerofftestvms.sh
+++ b/powerofftestvms.sh
@@ -19,9 +19,9 @@ fi
 
 get_test_vm_list # populate TEST_VM_LIST
 
-# TODO: Error out if no test VM's are available.
 if [ $(echo ${#TEST_VM_LIST[@]}) -eq 0 ]; then
   err "No test VMs configured in Satellite"
+  exit 1
 fi
 
 # shutdown test VMs

From aa4d8400350b5d89d0809b16e7b59037bafbe3cf Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 20:09:21 +0200
Subject: [PATCH 07/22] WIP, rename two pipeline stages

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 653a45f..6a574a2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -61,14 +61,14 @@ node {
   }
   executeStage(stagePubAndPromote, 'publish and promote CV')
 
-  def stagePrepVms = {
+  def stagePrepTestVms = {
     if (params.REBUILD_VMS == true) {
       executeScript("${SCRIPTS_DIR}/buildtestvms.sh")
     } else {
       executeScript("${SCRIPTS_DIR}/starttestvms.sh")
     }
   }
-  executeStage(stagePrepVms, 'prepare VMs')
+  executeStage(stagePrepTestVms, 'prepare test VMs')
 
   def stageRunTests = {
     executeScript("${SCRIPTS_DIR}/pushtests.sh")
@@ -76,14 +76,14 @@ node {
   }
   executeStage(stageRunTests, 'run tests')
    
-  def stagePowerOff = {
+  def stagePowerOffTestVMs = {
     if (params.POWER_OFF_VMS_AFTER_BUILD == true) {
       executeScript("${SCRIPTS_DIR}/powerofftestvms.sh")
     } else {
       println "test VMs are not shut down as per passed configuration"
     }
   }
-  executeStage(stagePowerOff, 'power off VMs')
+  executeStage(stagePowerOffTestVMs, 'power off test VMs')
 
   def stageCleanup = {
     executeScript("${SCRIPTS_DIR}/cleanup.sh")

From 5a57dd39c90b81e1f257e47cd4978992a1e8aac8 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 21:22:14 +0200
Subject: [PATCH 08/22] initial commit of wait4goldenvmsup.sh

---
 wait4goldenvmsup.sh | 52 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100755 wait4goldenvmsup.sh

diff --git a/wait4goldenvmsup.sh b/wait4goldenvmsup.sh
new file mode 100755
index 0000000..90bcf81
--- /dev/null
+++ b/wait4goldenvmsup.sh
@@ -0,0 +1,52 @@
+#!/bin/bash -x
+
+# Wait for golden VMs to be up (is run by pipeline before shutdowngoldenvms.sh)
+#
+# e.g ${WORKSPACE}/scripts/wait4goldenvmsup.sh 'test'
+#
+
+# Load common parameter variables
+. $(dirname "${0}")/common.sh
+
+get_golden_vm_list # populate GOLDEN_VM_LIST
+
+# If buildgoldenvms.sh ended cleanly but the VMs remain powered up, then
+# we need to wait until all the machines are up and can be ssh-ed to.
+# Only then will we tell the hypervisor (via Satellite) to shut down cleanly 
+# with a pipeline step shutdowngoldenvms.sh
+declare -A vmcopy # declare an associative array to copy our VM array into
+for I in "${GOLDEN_VM_LIST[@]}"; do vmcopy[$I]=$I; done
+
+WAIT=0
+while [[ ${#vmcopy[@]} -gt 0 ]]
+do
+    inform "Waiting 15 seconds"
+    sleep 15
+    ((WAIT+=15))
+    for I in "${vmcopy[@]}"
+    do
+        inform "Checking if golden VM $I has rebooted into OS before next pipeine step attemopts clean shutdown."
+        status=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer host info --name $I | \
+            grep -e \"Managed.*yes\" -e \"Enabled.*yes\" -e \"Build.*no\" \
+		| wc -l")
+        # Check if status is OK, ping reacts and SSH is there, then success!
+        if [[ ${status} == 3 ]] && ping -c 1 -q $I && nc -w 1 $I 22
+        then
+            tell "Success!"
+            unset vmcopy[$I]
+        else
+            tell "Not yet."
+        fi
+    done
+    if [[ ${WAIT} -gt 6000 ]]
+    then
+        err "Golden VM not reachable via ssh after 6000 seconds. Exiting."
+        exit 1
+    fi
+done
+
+# Wait another 30s to be on the safe side
+sleep 30
+
+# since a golden VM is meant to be imaged, do NOT do anything else here.
\ No newline at end of file

From 22b155c1f7ba154db8f7019d08bd9b30d0b3ece9 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 21:24:41 +0200
Subject: [PATCH 09/22] WIP, build the golden VMs and if needed shut down

---
 Jenkinsfile | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6a574a2..71b1f57 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -85,6 +85,24 @@ node {
   }
   executeStage(stagePowerOffTestVMs, 'power off test VMs')
 
+/*
+* promote to GOLDENVM_ENV here or do we do both test and golden VMs from same LCE?
+* the latter is actually fine as long as the pipeline exits on failure at one of the previous steps
+* the former gives us a nicer separation (so (C)CVs in the LCE can be used for other tasks that want only a version where automated testing passed)
+*/
+  def stagePromote2GoldenLCE = {
+    executeScript("${SCRIPTS_DIR}/promote2goldenlce.sh")
+    executeScript("${SCRIPTS_DIR}/capsule-sync-check.sh")
+  }
+  executeStage(stagePromote2GoldenLCE, 'promote CV to golden')
+
+  def stagePrepGoldenVms = {
+    executeScript("${SCRIPTS_DIR}/buildgoldenvms.sh")
+    executeScript("${SCRIPTS_DIR}/wait4goldenvmsup.sh")
+    executeScript("${SCRIPTS_DIR}/shutdowngoldenvms.sh")
+  }
+  executeStage(stagePrepGoldenVms, 'prepare golden VMs')
+
   def stageCleanup = {
     executeScript("${SCRIPTS_DIR}/cleanup.sh")
   }

From 8fe4d63b1fff51da00453426584a194c5aeffe60 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 22:20:15 +0200
Subject: [PATCH 10/22] WIP, add script to promote to golden LCE

---
 Jenkinsfile          |  4 ++++
 promote2goldenlce.sh | 52 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100755 promote2goldenlce.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index 71b1f57..6c0ac2d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -94,7 +94,11 @@ node {
     executeScript("${SCRIPTS_DIR}/promote2goldenlce.sh")
     executeScript("${SCRIPTS_DIR}/capsule-sync-check.sh")
   }
+<<<<<<< HEAD
   executeStage(stagePromote2GoldenLCE, 'promote CV to golden')
+=======
+  executeStage(stagePromote2GoldenLCE, 'publish and promote CV')
+>>>>>>> 8ea8585... WIP, add script to promote to golden LCE
 
   def stagePrepGoldenVms = {
     executeScript("${SCRIPTS_DIR}/buildgoldenvms.sh")
diff --git a/promote2goldenlce.sh b/promote2goldenlce.sh
new file mode 100755
index 0000000..feee549
--- /dev/null
+++ b/promote2goldenlce.sh
@@ -0,0 +1,52 @@
+#!/bin/bash -x
+
+# promote the content view(s) from TESTVM_ENV to GOLDENVM_ENV
+
+# Load common parameter variables
+. $(dirname "${0}")/common.sh
+
+# If MODIFIED_CONTENT_FILE is not 0 bytes, then publishcv.sh has
+# attempted a (C)CV publish step plus a promotion to LCE TESTVM_ENV
+# thus we can promote to GOLDENVM_ENV now
+# (since this script is called by a pipelline step that is only executed if the prior steps did NOT fail)
+if [[ ! -s "${MODIFIED_CONTENT_FILE}" ]]
+then
+    echo "No entries in ${MODIFIED_CONTENT_FILE} no need to continue with $0"
+    exit 0
+fi
+
+# Create an array from all the content view names
+oldIFS="${IFS}"
+i=0
+IFS=','
+for cv in ${CV} ${CV_PASSIVE_LIST}
+do
+        CV_LIST[$i]="${cv}"
+        ((i++))
+done
+IFS="${oldIFS}"
+
+if [[ -n ${GOLDENVM_ENV} ]]
+then
+    for (( i = 0; i < ${#CV_LIST[@]}; i++ ))
+    do # promote the latest version of each CV
+        cv=${CV_LIST[$i]}
+        ver_id=${VER_ID_LIST[$i]}
+
+        inform "Promoting version ${ver_id} of ${cv} to LCE ${GOLDENVM_ENV}"
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+        "hammer content-view version promote --content-view \"${cv}\" --organization \"${ORG}\" \
+        --to-lifecycle-environment-id \"${GOLDENVM_ENV}\" --force --id ${ver_id}"
+    done
+
+    # we also promote the latest version of each CCV
+    for ccv_id in ${CCV_IDS[@]}
+    do
+        ccv_ver=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+            "hammer --csv content-view version list --content-view-id ${ccv_id} --organization \"${ORG}\"" | awk -F',' '$1 ~ /^[0-9]+$/ {if ($3 > maxver) {maxver = $3; maxid = $1} } END {print maxid}')
+        inform "Promoting version ${ccv_ver} of CCV ID ${ccv_id} to LCE ${GOLDENVM_ENV}"
+        ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+        "hammer content-view version promote --content-view-id \"${ccv_id}\" --organization \"${ORG}\" \
+        --to-lifecycle-environment-id \"${GOLDENVM_ENV}\" --force --id ${ccv_ver}"
+    done
+fi

From 4be86e857b5b61f3ca2a213b10fb45b931c6fe2f Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Tue, 6 Oct 2020 23:51:20 +0200
Subject: [PATCH 11/22] fixup, which HC to use for golden VMs

---
 rhel-8-script-env-vars-puppet-only.groovy | 1 +
 rhel-8-script-env-vars-rpm.groovy         | 1 +
 2 files changed, 2 insertions(+)

diff --git a/rhel-8-script-env-vars-puppet-only.groovy b/rhel-8-script-env-vars-puppet-only.groovy
index 92e3253..f51da0e 100644
--- a/rhel-8-script-env-vars-puppet-only.groovy
+++ b/rhel-8-script-env-vars-puppet-only.groovy
@@ -1,6 +1,7 @@
 env.REPO_ID="3825"
 env.PUPPET_REPO_ID="9"
 env.TESTVM_HOSTCOLLECTION="hc-soe-el8-test"
+env.GOLDENVM_HOSTCOLLECTION="hc-soe-el8-golden"
 env.PUPPET_REPO="/var/www/html/pub/soe-puppet-only"
 env.CV="cv-puppet-only"
 env.CV_PASSIVE_LIST=""
diff --git a/rhel-8-script-env-vars-rpm.groovy b/rhel-8-script-env-vars-rpm.groovy
index 8f847ec..f2adcc3 100644
--- a/rhel-8-script-env-vars-rpm.groovy
+++ b/rhel-8-script-env-vars-rpm.groovy
@@ -2,6 +2,7 @@
 env.REPO_ID="3825"
 env.PUPPET_REPO_ID="8"
 env.TESTVM_HOSTCOLLECTION="hc-soe-el8-test"
+env.GOLDENVM_HOSTCOLLECTION="hc-soe-el8-golden"
 env.YUM_REPO="/var/www/html/pub/soe-repo/rhel8"
 env.PUPPET_REPO="/var/www/html/pub/soe-puppet"
 env.CV="cv-soe-ci-el8"

From 4b8edb9d099fafbfe56d0b56974e8723646550eb Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 11:47:16 +0200
Subject: [PATCH 12/22] comment, who does sysprep?

---
 Jenkinsfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 6c0ac2d..3d91c2f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -107,6 +107,8 @@ node {
   }
   executeStage(stagePrepGoldenVms, 'prepare golden VMs')
 
+  // where do we run virt-sysprep (1) after this is successful? Ideally on the machine doing qemu-img convert
+
   def stageCleanup = {
     executeScript("${SCRIPTS_DIR}/cleanup.sh")
   }

From 5385ab6322edb78a31364c32cda7174c4dc6a50b Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 11:47:55 +0200
Subject: [PATCH 13/22] WIP, populate VER_ID_LIST

---
 promote2goldenlce.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/promote2goldenlce.sh b/promote2goldenlce.sh
index feee549..3ac5d77 100755
--- a/promote2goldenlce.sh
+++ b/promote2goldenlce.sh
@@ -26,6 +26,16 @@ do
 done
 IFS="${oldIFS}"
 
+# Get a list of all CV version IDs
+for cv in "${CV_LIST[@]}"
+do
+    # get the latest version of each CV, add it to the array
+    inform "Get the latest version of CV ${cv}"
+    VER_ID_LIST+=( "$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
+	"hammer content-view info --name \"${cv}\" --organization \"${ORG}\" \
+	| sed -n \"/Versions:/,/Components:/p\" | grep \"ID:\" | tr -d ' ' | cut -f2 -d ':' | sort -n | tail -n 1")" )
+done
+
 if [[ -n ${GOLDENVM_ENV} ]]
 then
     for (( i = 0; i < ${#CV_LIST[@]}; i++ ))

From e05fc1408604e56e2c99540f9ad78792fbf4af78 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 12:53:01 +0200
Subject: [PATCH 14/22] we promote a version ID, not a version number

---
 promote2goldenlce.sh | 2 +-
 publishcv.sh         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/promote2goldenlce.sh b/promote2goldenlce.sh
index 3ac5d77..f294bb5 100755
--- a/promote2goldenlce.sh
+++ b/promote2goldenlce.sh
@@ -43,7 +43,7 @@ then
         cv=${CV_LIST[$i]}
         ver_id=${VER_ID_LIST[$i]}
 
-        inform "Promoting version ${ver_id} of ${cv} to LCE ${GOLDENVM_ENV}"
+        inform "Promoting version ID ${ver_id} of ${cv} to LCE ${GOLDENVM_ENV}"
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
         "hammer content-view version promote --content-view \"${cv}\" --organization \"${ORG}\" \
         --to-lifecycle-environment-id \"${GOLDENVM_ENV}\" --force --id ${ver_id}"
diff --git a/publishcv.sh b/publishcv.sh
index 609eba4..e0df465 100755
--- a/publishcv.sh
+++ b/publishcv.sh
@@ -123,7 +123,7 @@ then
         cv=${CV_LIST[$i]}
         ver_id=${VER_ID_LIST[$i]}
 
-        inform "Promoting version ${ver_id} of ${cv} to LCE ${TESTVM_ENV}"
+        inform "Promoting version ID ${ver_id} of ${cv} to LCE ${TESTVM_ENV}"
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
         "hammer content-view version promote --content-view \"${cv}\" --organization \"${ORG}\" \
         --to-lifecycle-environment-id \"${TESTVM_ENV}\" --force --id ${ver_id}"

From d3615be204973d47f729be2c819ba91c5ba5252d Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 14:50:58 +0200
Subject: [PATCH 15/22] golden VM needs time after build switched to no

---
 buildgoldenvms.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
index 838ee79..cc3aba8 100755
--- a/buildgoldenvms.sh
+++ b/buildgoldenvms.sh
@@ -121,3 +121,5 @@ do
     fi
 done
 
+inform "A host that exited build mode is given 3 minutes to finish anaconda ceanly"
+sleep 180
\ No newline at end of file

From 84605af34124f3dfd311f2bf78a332c9fb5834d8 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 17:12:33 +0200
Subject: [PATCH 16/22] WIP, this should stop on failed tests

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3d91c2f..1c40445 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,7 +72,7 @@ node {
 
   def stageRunTests = {
     executeScript("${SCRIPTS_DIR}/pushtests.sh")
-    step([$class: "TapPublisher", testResults: "test_results/*.tap", ])
+    step([$class: "TapPublisher", testResults: "test_results/*.tap", failedTestsMarkBuildAsFailure: true, ])
   }
   executeStage(stageRunTests, 'run tests')
    

From ce33ced0f3bca009636158a0777182f267296d2b Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Wed, 7 Oct 2020 19:52:39 +0200
Subject: [PATCH 17/22] pipelin to stop on failed tests

Kudos to Janine for help on this.
---
 Jenkinsfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1c40445..cf68349 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,7 +72,11 @@ node {
 
   def stageRunTests = {
     executeScript("${SCRIPTS_DIR}/pushtests.sh")
-    step([$class: "TapPublisher", testResults: "test_results/*.tap", failedTestsMarkBuildAsFailure: true, ])
+    step([$class: "TapPublisher", testResults: "test_results/*.tap", failedTestsMarkBuildAsFailure: true ])
+    if (currentBuild.result == 'FAILURE') {
+      isInErrorState = true
+      error('There were test failures')
+    }
   }
   executeStage(stageRunTests, 'run tests')
    

From 67204030e2dc0c4a83912ffe263971e3fe0ee050 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Thu, 8 Oct 2020 11:05:30 +0200
Subject: [PATCH 18/22] fixup, better name for stage promote to golden

---
 Jenkinsfile | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index cf68349..da11bc0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -98,11 +98,7 @@ node {
     executeScript("${SCRIPTS_DIR}/promote2goldenlce.sh")
     executeScript("${SCRIPTS_DIR}/capsule-sync-check.sh")
   }
-<<<<<<< HEAD
   executeStage(stagePromote2GoldenLCE, 'promote CV to golden')
-=======
-  executeStage(stagePromote2GoldenLCE, 'publish and promote CV')
->>>>>>> 8ea8585... WIP, add script to promote to golden LCE
 
   def stagePrepGoldenVms = {
     executeScript("${SCRIPTS_DIR}/buildgoldenvms.sh")

From 2dd05aec623fb8f1bc00054ccd210c5f0384ece6 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Thu, 22 Oct 2020 15:39:37 +0200
Subject: [PATCH 19/22] address #128

---
 buildtestvms.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/buildtestvms.sh b/buildtestvms.sh
index 879af70..631de59 100755
--- a/buildtestvms.sh
+++ b/buildtestvms.sh
@@ -27,11 +27,11 @@ fi
 # rebuild test VMs
 for I in "${TEST_VM_LIST[@]}"
 do
-    inform "Rebuilding VM ID $I"
+    inform "Rebuilding VM $I"
     ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-        "hammer host update --id $I --build yes"
+        "hammer host update --name $I --build yes"
 
-    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --id $I" | grep Power | cut -f2 -d: | tr -d ' ')
+    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --name $I" | grep Power | cut -f2 -d: | tr -d ' ')
 
     # different hypervisors report power status with different words. parse and get a single word per status
     # KVM uses running / shutoff
@@ -69,13 +69,13 @@ do
     then
         # forcefully poweroff the SUT
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host stop --force --id $I"
+            "hammer host stop --force --name $I"
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host start --id $I"
+            "hammer host start --name $I"
     elif [[ ${_STATUS} == 'Off' ]]
     then
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host start --id $I"
+            "hammer host start --name $I"
     else
         err "Host $I is neither running nor shutoff. No action possible!"
         exit 1

From 5248c047384cbdeddaa558b8279e0d51434574cb Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Thu, 22 Oct 2020 15:47:19 +0200
Subject: [PATCH 20/22] buildgoldenvms also needs the fix for issue 128

---
 buildgoldenvms.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
index cc3aba8..24cbdbf 100755
--- a/buildgoldenvms.sh
+++ b/buildgoldenvms.sh
@@ -29,9 +29,9 @@ for I in "${GOLDEN_VM_LIST[@]}"
 do
     inform "Rebuilding VM ID $I"
     ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-        "hammer host update --id $I --build yes"
+        "hammer host update --name $I --build yes"
 
-    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --id $I" | grep Power | cut -f2 -d: | tr -d ' ')
+    _PROBED_STATUS=$(ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} "hammer host status --name $I" | grep Power | cut -f2 -d: | tr -d ' ')
 
     # different hypervisors report power status with different words. parse and get a single word per status
     # KVM uses running / shutoff
@@ -69,13 +69,13 @@ do
     then
         # forcefully poweroff the SUT
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host stop --force --id $I"
+            "hammer host stop --force --name $I"
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host start --id $I"
+            "hammer host start --name $I"
     elif [[ ${_STATUS} == 'Off' ]]
     then
         ssh -q -l ${PUSH_USER} -i ${RSA_ID} ${SATELLITE} \
-            "hammer host start --id $I"
+            "hammer host start --name $I"
     else
         err "Host $I is neither running nor shutoff. No action possible!"
         exit 1
@@ -121,5 +121,5 @@ do
     fi
 done
 
-inform "A host that exited build mode is given 3 minutes to finish anaconda ceanly"
-sleep 180
\ No newline at end of file
+inform "A host that exited build mode is given 3 minutes to finish anaconda cleanly"
+sleep 180

From 4e5bf7e1932ff3a7b100bc9a584f6e5a9633f81d Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Thu, 22 Oct 2020 17:32:08 +0200
Subject: [PATCH 21/22] =?UTF-8?q?fixup,=20zu=20Guttenberg=20error,=20test?=
 =?UTF-8?q?=20=E2=86=92=20golden?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 buildgoldenvms.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
index 24cbdbf..5636595 100755
--- a/buildgoldenvms.sh
+++ b/buildgoldenvms.sh
@@ -19,12 +19,12 @@ fi
 
 get_golden_vm_list # populate GOLDEN_VM_LIST
 
-# TODO: Error out if no test VM's are available.
+# TODO: Error out if no golden VM's are available.
 if [ $(echo ${#GOLDEN_VM_LIST[@]}) -eq 0 ]; then
-  err "No test VMs configured in Satellite"
+  err "No golden VMs configured in Satellite"
 fi
 
-# rebuild test VMs
+# rebuild golden VMs
 for I in "${GOLDEN_VM_LIST[@]}"
 do
     inform "Rebuilding VM ID $I"
@@ -83,7 +83,7 @@ do
 done
 
 
-# we need to wait until all the test machines have been rebuilt by foreman
+# we need to wait until all the golden machines have been rebuilt by foreman
 # this check was previously only in pushtests, but when using pipelines 
 # it's more sensible to wait here while the machines are in build mode
 # the ping and ssh checks must remain in pushtests.sh

From 8a32a397d2f0032b37fc1788bb7cd960e08a98f8 Mon Sep 17 00:00:00 2001
From: "Patrick C. F. Ernzer" <pcfe@pcfe.net>
Date: Mon, 16 Nov 2020 17:18:13 +0100
Subject: [PATCH 22/22] show power status in error msg

---
 buildgoldenvms.sh | 2 +-
 buildtestvms.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildgoldenvms.sh b/buildgoldenvms.sh
index 5636595..2556f6f 100755
--- a/buildgoldenvms.sh
+++ b/buildgoldenvms.sh
@@ -62,7 +62,7 @@ do
         _STATUS=Off
         ;;
       *)
-        echo "can not parse power status, please review $0"
+        echo "can not parse power status, please review $0 for status ${_PROBED_STATUS}"
     esac
 
     if [[ ${_STATUS} == 'On' ]]
diff --git a/buildtestvms.sh b/buildtestvms.sh
index 631de59..20f9bc7 100755
--- a/buildtestvms.sh
+++ b/buildtestvms.sh
@@ -62,7 +62,7 @@ do
         _STATUS=Off
         ;;
       *)
-        echo "can not parse power status, please review $0"
+        echo "can not parse power status, please review $0 for status ${_PROBED_STATUS}"
     esac
 
     if [[ ${_STATUS} == 'On' ]]