Run lantern parallel tests across update versions

scripts/run_all_tests.sh

@@ -6,9 +6,15 @@ PSQL=psql
 TMP_ROOT=/tmp/lantern
 TMP_OUTDIR=$TMP_ROOT/tmp_output
 FILTER="${FILTER:-}"
+EXCLUDE="${EXCLUDE:-}"
 # $USER is not set in docker containers, so use whoami
 DEFAULT_USER=$(whoami)
 
+if [[ -n "$FILTER" && -n "$EXCLUDE" ]]; then
+    echo "-FILTER and -EXCLUDE cannot be used together, please use only one"
+    exit 1
+fi
+
 # typically default user is root in a docker container
 # and in those cases postgres is the user with appropriate permissions
 # to the database
@@ -78,28 +84,54 @@ while [[ "$#" -gt 0 ]]; do
     shift
 done
 
+FIRST_TEST=1
+function print_test {
+    if [ "$PARALLEL" -eq 1 ]; then
+        if [ $1 == end ]; then
+            echo -e "\ntest: $1" >> $2
+        elif [ $1 == begin ]; then
+            echo -e "\ntest: $1" >> $2
+        else
+            if [ "$FIRST_TEST" -eq 1 ]; then
+                echo -n "test: $1" >> $2
+                FIRST_TEST=0
+            else
+                echo -n " $1" >> $2
+            fi
+        fi
+    else
+        echo "test: $1" >> $2
+    fi
+}
+
 # Generate schedule.txt
 rm -rf $TMP_OUTDIR/schedule.txt
 if [ "$PARALLEL" -eq 1 ]; then
     SCHEDULE='parallel_schedule.txt'
 else
     SCHEDULE='schedule.txt'
 fi
-if [ -n "$FILTER" ]; then
+if [[ -n "$FILTER" || -n "$EXCLUDE" ]]; then
     if [ "$PARALLEL" -eq 1 ]; then
     	TEST_FILES=$(cat $SCHEDULE | grep -E '^(test:|test_begin:|test_end:)' | sed -E -e 's/^test:|test_begin:|test_end://' | tr " " "\n" | sed -e '/^$/d')
     else
-	    if [[ "$pgvector_installed" == "1" ]]; then
-		TEST_FILES=$(cat $SCHEDULE | grep -E '^(test:|test_pgvector:)' | sed -E -e 's/^test:|test_pgvector://' | tr " " "\n" | sed -e '/^$/d')
-	    else
-		TEST_FILES=$(cat $SCHEDULE | grep '^test:' | sed -e 's/^test://' | tr " " "\n" | sed -e '/^$/d')
-	    fi
+        if [[ "$pgvector_installed" == "1" ]]; then
+            TEST_FILES=$(cat $SCHEDULE | grep -E '^(test:|test_pgvector:)' | sed -E -e 's/^test:|test_pgvector://' | tr " " "\n" | sed -e '/^$/d')
+        else
+            TEST_FILES=$(cat $SCHEDULE | grep '^test:' | sed -e 's/^test://' | tr " " "\n" | sed -e '/^$/d')
+        fi
     fi
 
     while IFS= read -r f; do
-        if [[ $f == *"$FILTER"* ]]; then
-            echo "HERE $f"
-            echo "test: $f" >> $TMP_OUTDIR/schedule.txt
+        if [ -n "$FILTER" ]; then
+            if [[ $f == *"$FILTER"* ]]; then
+                print_test $f $TMP_OUTDIR/schedule.txt $FIRST_TEST
+            fi
+        elif [ -n "$EXCLUDE" ]; then
+            if [[ $f == *"$EXCLUDE"* ]]; then
+                continue
+            fi
+            print_test $f $TMP_OUTDIR/schedule.txt $FIRST_TEST
         fi
     done <<< "$TEST_FILES"
 

scripts/test_updates.py

@@ -34,6 +34,8 @@ def update_from_tag(from_version: str, to_version: str):
     repo.git.checkout(sha_before)
     res = subprocess.run(f"cd {args.builddir} ; git submodule update && cmake .. && make -j4 && make install && make test", shell=True)
     res = subprocess.run(f"cd {args.builddir} ; UPDATE_EXTENSION=1 UPDATE_FROM={from_version} UPDATE_TO={to_version} make test", shell=True)
+    res = subprocess.run(f"cd {args.builddir} ; UPDATE_EXTENSION=1 UPDATE_FROM={from_version} UPDATE_TO={from_version} make test-parallel FILTER=begin", shell=True)
+    res = subprocess.run(f"cd {args.builddir} ; UPDATE_EXTENSION=1 UPDATE_FROM={from_version} UPDATE_TO={to_version} make test-parallel EXCLUDE=begin", shell=True)
     #todo:: run query and check portion of parallel tests
 
 def incompatible_version(pg_version, version_tag):

test/test_runner.sh

@@ -46,17 +46,16 @@ function run_regression_test {
                perl -nle'print if !m{DEBUG:(?!.*LANTERN)}'
 }
 
-trap drop_db EXIT
-
-
 
 
 # Change directory to sql so sql imports will work correctly
 cd sql/
 
 # install lantern extension
-psql "$@" -U ${DB_USER} -d postgres -v ECHO=none -q -c "DROP DATABASE IF EXISTS ${TEST_CASE_DB};" 2>/dev/null
-psql "$@" -U ${DB_USER} -d postgres -v ECHO=none -q -c "CREATE DATABASE ${TEST_CASE_DB};" 2>/dev/null
+if [[ "$PARALLEL" -eq 0 || "$TESTFILE_NAME" == "begin" ]]; then
+     psql "$@" -U ${DB_USER} -d postgres -v ECHO=none -q -c "DROP DATABASE IF EXISTS ${TEST_CASE_DB};" 2>/dev/null
+     psql "$@" -U ${DB_USER} -d postgres -v ECHO=none -q -c "CREATE DATABASE ${TEST_CASE_DB};" 2>/dev/null
+fi
 if [ ! -z "$UPDATE_EXTENSION" ]
 then
      if [ -z "$UPDATE_FROM" ] || [ -z "$UPDATE_TO" ]
@@ -66,7 +65,7 @@ then
      fi
 
      # install the old version of the extension and sanity-check that all tests pass
-     psql "$@" -U ${DB_USER} -d ${TEST_CASE_DB} -v ECHO=none -q -c "SET client_min_messages=error; CREATE EXTENSION lantern VERSION '$UPDATE_FROM';" 2>/dev/null
+     psql "$@" -U ${DB_USER} -d ${TEST_CASE_DB} -v ECHO=none -q -c "SET client_min_messages=error; CREATE EXTENSION IF NOT EXISTS lantern VERSION '$UPDATE_FROM';" 2>/dev/null
      psql "$@" -U ${DB_USER} -d ${TEST_CASE_DB} -v ECHO=none -q -f utils/common.sql 2>/dev/null
      run_regression_test $@
      # upgrade to the new version of the extension and make sure that all existing tests still pass