From 2959b93c7ed841b58480acccd005613803588295 Mon Sep 17 00:00:00 2001
From: Johannes <jkbecker@bu.edu>
Date: Tue, 6 Oct 2020 00:38:43 -0400
Subject: [PATCH 1/4] initial automatic recipe testing script

---
 .autotest.sh | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 .autotest.sh

diff --git a/.autotest.sh b/.autotest.sh
new file mode 100644
index 0000000..83b3d4b
--- /dev/null
+++ b/.autotest.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Pull most recent master into this branch
+
+git pull origin master -q
+dimage="pybombs/pybombs-prefix"
+dtag="latest"
+drun="docker run --rm -it -v $(pwd):/root/.pybombs/recipes/gr-recipes $dimage:$dtag"
+
+pbversion=`$drun pybombs --version`
+pyversion=`$drun python --version`
+imgversion=`docker images --filter=reference="$dimage" --format={{.Repository}}:{{.Tag}}:{{.Digest}} | grep $dtag | sed "s/^.*sha256://"`
+
+for recipefile in *.lwr; do
+	rname=`echo $recipefile | sed "s/.lwr$//"`
+	rversion=`git log $recipefile | head -n 1 | sed "s/commit //"`
+	echo "Testing installation of $rname"
+	$drun pybombs -vy install $rname > .autotest.log
+	if [ $? -eq 0 ]; then
+    	echo $rname OK
+	else
+		# Installation failed
+		issuetitle="Error installing $rname@${rversion:0:7} (autotest)"
+		echo ".autotest.sh found an issue while attempting to install **[$rname](../../blob/master/$recipefile)** (last changed in commit [#${rversion:0:7}](https://github.com/gnuradio/gr-recipes/commit/$rversion))." > .autotest.issue
+		echo "" >> .autotest.issue
+		echo "## Setup used:" >> .autotest.issue
+		echo "" >> .autotest.issue
+		echo " - Docker image: $dimage:$dtag ([Explore](https://hub.docker.com/layers/$dimage/$dtag/images/sha256-$imgversion?context=explore) | [Tags](https://hub.docker.com/r/$dimage/tags))" >> .autotest.issue
+		echo " - PyBOMBS version: $pbversion" >> .autotest.issue
+		echo " - Python version: $pyversion" >> .autotest.issue
+		echo "" >> .autotest.issue
+		echo "## Install log:" >> .autotest.issue
+		echo '```bash' >> .autotest.issue
+		cat .autotest.log >> .autotest.issue
+		echo '```' >> .autotest.issue
+		echo $issuetitle
+		cat .autotest.issue
+	fi
+	echo ""
+done
+
+

From 865ca647184a08b512b4af2cce0ec238066d7a57 Mon Sep 17 00:00:00 2001
From: Johannes <jkbecker@bu.edu>
Date: Mon, 19 Oct 2020 16:28:20 -0400
Subject: [PATCH 2/4] autotest script, manual mode (no automatic issue
 creation)

---
 .autotest.sh | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/.autotest.sh b/.autotest.sh
index 83b3d4b..2e96486 100644
--- a/.autotest.sh
+++ b/.autotest.sh
@@ -1,23 +1,26 @@
 #!/bin/bash
 
 # Pull most recent master into this branch
+git pull origin master
 
-git pull origin master -q
+# Define some important values
 dimage="pybombs/pybombs-prefix"
 dtag="latest"
 drun="docker run --rm -it -v $(pwd):/root/.pybombs/recipes/gr-recipes $dimage:$dtag"
-
 pbversion=`$drun pybombs --version`
 pyversion=`$drun python --version`
 imgversion=`docker images --filter=reference="$dimage" --format={{.Repository}}:{{.Tag}}:{{.Digest}} | grep $dtag | sed "s/^.*sha256://"`
 
-for recipefile in *.lwr; do
+# Testing function for an individual recipe
+function run_test () {
+	recipefile=${1:-}
 	rname=`echo $recipefile | sed "s/.lwr$//"`
 	rversion=`git log $recipefile | head -n 1 | sed "s/commit //"`
-	echo "Testing installation of $rname"
+	echo "    Testing installation of $rname ($recipefile @ ${rversion:0:7})"
 	$drun pybombs -vy install $rname > .autotest.log
 	if [ $? -eq 0 ]; then
-    	echo $rname OK
+		# Installation succeeded, nothing to do
+    		echo "        $rname OK"
 	else
 		# Installation failed
 		issuetitle="Error installing $rname@${rversion:0:7} (autotest)"
@@ -33,10 +36,21 @@ for recipefile in *.lwr; do
 		echo '```bash' >> .autotest.issue
 		cat .autotest.log >> .autotest.issue
 		echo '```' >> .autotest.issue
-		echo $issuetitle
+		echo "        $issuetitle"
 		cat .autotest.issue
 	fi
 	echo ""
-done
+}
 
+# Run tests either on all recipes in the directory, or on an individual recipe
+# provided as an arument.
+if [ $# -eq 0 ]; then
+	echo "Running install tests on all files in `pwd`..."
+	for recipefile in *.lwr; do
+		run_test $recipefile
+	done
+else
+	echo "Running install tests on $1..."
+	run_test $1
+fi
 

From 1c9ba14bdfa42410aca709faad5f2f36f379b6e4 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Mon, 19 Oct 2020 22:21:55 -0400
Subject: [PATCH 3/4] functional autotest script as described in #200

---
 .autotest.sh | 75 +++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 21 deletions(-)

diff --git a/.autotest.sh b/.autotest.sh
index 2e96486..1fdc97c 100644
--- a/.autotest.sh
+++ b/.autotest.sh
@@ -6,47 +6,80 @@ git pull origin master
 # Define some important values
 dimage="pybombs/pybombs-prefix"
 dtag="latest"
-drun="docker run --rm -it -v $(pwd):/root/.pybombs/recipes/gr-recipes $dimage:$dtag"
-pbversion=`$drun pybombs --version`
-pyversion=`$drun python --version`
+drun="docker run --rm -it -v $(pwd):/root/.pybombs/recipes/gr-recipes -v $(pwd)/.build:/pybombs/src: $dimage:$dtag"
+pbversion=`$drun pybombs --version | tr -d '\r'`
+pyversion=`$drun python3 --version | tr -d '\r'`
 imgversion=`docker images --filter=reference="$dimage" --format={{.Repository}}:{{.Tag}}:{{.Digest}} | grep $dtag | sed "s/^.*sha256://"`
 
+function autotest_reset () {
+	mkdir -p .build
+	rm -rf .build/*
+
+}
+
+function autotest_clean () {
+	rm -f .autotest_*.issue
+	rm -f .autotest_*.log
+	rm -rf .build
+}
+
 # Testing function for an individual recipe
 function run_test () {
 	recipefile=${1:-}
 	rname=`echo $recipefile | sed "s/.lwr$//"`
 	rversion=`git log $recipefile | head -n 1 | sed "s/commit //"`
-	echo "    Testing installation of $rname ($recipefile @ ${rversion:0:7})"
-	$drun pybombs -vy install $rname > .autotest.log
+	printf "    Testing installation of $rname ($recipefile @ ${rversion:0:7}) ... "
+	$drun pybombs -vy install $rname > .autotest_${rname}.log
 	if [ $? -eq 0 ]; then
 		# Installation succeeded, nothing to do
-    		echo "        $rname OK"
+    		echo "OK"
 	else
 		# Installation failed
-		issuetitle="Error installing $rname@${rversion:0:7} (autotest)"
-		echo ".autotest.sh found an issue while attempting to install **[$rname](../../blob/master/$recipefile)** (last changed in commit [#${rversion:0:7}](https://github.com/gnuradio/gr-recipes/commit/$rversion))." > .autotest.issue
-		echo "" >> .autotest.issue
-		echo "## Setup used:" >> .autotest.issue
-		echo "" >> .autotest.issue
-		echo " - Docker image: $dimage:$dtag ([Explore](https://hub.docker.com/layers/$dimage/$dtag/images/sha256-$imgversion?context=explore) | [Tags](https://hub.docker.com/r/$dimage/tags))" >> .autotest.issue
-		echo " - PyBOMBS version: $pbversion" >> .autotest.issue
-		echo " - Python version: $pyversion" >> .autotest.issue
-		echo "" >> .autotest.issue
-		echo "## Install log:" >> .autotest.issue
-		echo '```bash' >> .autotest.issue
-		cat .autotest.log >> .autotest.issue
-		echo '```' >> .autotest.issue
-		echo "        $issuetitle"
-		cat .autotest.issue
+		issuetopic="$rname@${rversion:0:7} (PyBOMBS v$pbversion, Python v$pyversion)"
+		echo "FAILED"
+    	echo "            $issuetopic installation error"
+		issuehash=`echo "$issuetopic" | md5sum | cut -c1-5`
+		gh issue list --limit 999 | grep "autotest:$issuehash" > /dev/null 2>&1
+		if [ $? -eq 0 ]; then
+			printf "            Issue is known:\n              -> Issue "
+			gh issue list --limit 999 | grep "autotest:$issuehash"
+		else
+			issuetitle="Error installing $issuetopic [autotest:$issuehash]"
+			echo ".autotest.sh found an issue while attempting to install **[$rname](../../blob/master/$recipefile)** (last changed in commit [#${rversion:0:7}](https://github.com/gnuradio/gr-recipes/commit/$rversion))." >> .autotest_${rname}.issue
+			echo "" > .autotest_${rname}.issue
+			echo "## Setup used:" >> .autotest_${rname}.issue
+			echo "" >> .autotest_${rname}.issue
+			echo " - Docker image: $dimage:$dtag ([Explore](https://hub.docker.com/layers/$dimage/$dtag/images/sha256-$imgversion?context=explore) | [Tags](https://hub.docker.com/r/$dimage/tags))" >> .autotest_${rname}.issue
+			echo " - PyBOMBS version: $pbversion" >> .autotest_${rname}.issue
+			echo " - Python version: $pyversion" >> .autotest_${rname}.issue
+			echo "" >> .autotest_${rname}.issue
+			echo "## Install details:" >> .autotest_${rname}.issue
+			gistinput="gh gist create .autotest_${rname}.log"
+			if [ -f ".build/${rname}/build/CMakeFiles/CMakeError.log" ]; then
+				gistinput="${gistinput} .build/${rname}/build/CMakeFiles/CMakeError.log"
+			fi
+			if [ -f ".build/${rname}/build/CMakeFiles/CMakeOutput.log" ]; then
+				gistinput="${gistinput} .build/${rname}/build/CMakeFiles/CMakeOutput.log"
+			fi
+			gistoutput=`$gistinput 2> /dev/null`
+			echo $gistoutput >> .autotest_${rname}.issue
+			echo "## Meta:" >> .autotest_${rname}.issue
+			echo "This report has been created using the autotest script described in #200." >> .autotest_${rname}.issue
+			export issuebody=$(cat .autotest_${rname}.issue)
+			issuecreated=`gh issue create -t "${issuetitle}" -b "$issuebody" 2> /dev/null`
+			echo "              -> Issue created: $issuecreated"
+		fi
 	fi
 	echo ""
 }
 
 # Run tests either on all recipes in the directory, or on an individual recipe
 # provided as an arument.
+autotest_clean >/dev/null 2>&1
 if [ $# -eq 0 ]; then
 	echo "Running install tests on all files in `pwd`..."
 	for recipefile in *.lwr; do
+		autotest_reset >/dev/null 2>&1
 		run_test $recipefile
 	done
 else

From d425ebc569640c3d7ad1e4465feed5f6be5aeada Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Mon, 19 Oct 2020 22:52:29 -0400
Subject: [PATCH 4/4] more concise file reference, output fix.

---
 .autotest.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.autotest.sh b/.autotest.sh
index 1fdc97c..050845d 100644
--- a/.autotest.sh
+++ b/.autotest.sh
@@ -45,8 +45,8 @@ function run_test () {
 			gh issue list --limit 999 | grep "autotest:$issuehash"
 		else
 			issuetitle="Error installing $issuetopic [autotest:$issuehash]"
-			echo ".autotest.sh found an issue while attempting to install **[$rname](../../blob/master/$recipefile)** (last changed in commit [#${rversion:0:7}](https://github.com/gnuradio/gr-recipes/commit/$rversion))." >> .autotest_${rname}.issue
-			echo "" > .autotest_${rname}.issue
+			echo ".autotest.sh found an issue while attempting to install $rname from **[$recipefile @ commit ${rversion:0:7}](https://github.com/gnuradio/gr-recipes/blob/$rversion/$recipefile)** ([commit diff for ${rversion:0:7}](https://github.com/gnuradio/gr-recipes/commit/$rversion))." >> .autotest_${rname}.issue
+			echo "" >> .autotest_${rname}.issue
 			echo "## Setup used:" >> .autotest_${rname}.issue
 			echo "" >> .autotest_${rname}.issue
 			echo " - Docker image: $dimage:$dtag ([Explore](https://hub.docker.com/layers/$dimage/$dtag/images/sha256-$imgversion?context=explore) | [Tags](https://hub.docker.com/r/$dimage/tags))" >> .autotest_${rname}.issue