tests: re-organize the perf test suite (#13672)

* tests:new perf test to install many snaps with --no-wait and check changes

Install different snaps many times based on arch availability. In this
scenario all the snaps are installed in parallel and the test checks
the system can handel all the installs and remains active.

* performance tests reorganization

To avoid executing perf tests when the nested tests are executed, the
tests have been moved under tests/perf

tests/perf/main -> suite with the tests executed in the target device
tests/perf/nested -> suite with the tests executed in a nested
environment

* Update tests/perf/nested/install-many-snaps-no-wait/task.yaml

Co-authored-by: Miguel Pires <[email protected]>

* Update tests/perf/nested/install-many-snaps-no-wait/task.yaml

Co-authored-by: Miguel Pires <[email protected]>

* Update tests/perf/main/install-many-snaps-no-wait/task.yaml

Co-authored-by: Miguel Pires <[email protected]>

* improve logs check and fixes based on review comments

* addressing review comments

---------

Co-authored-by: Miguel Pires <[email protected]>

spread.yaml

@@ -1069,7 +1069,7 @@ suites:
         restore: |
             "$TESTSLIB"/prepare-restore.sh --restore-suite
 
-    tests/perf/:
+    tests/perf/main/:
         summary: Performance and Load tests
         backends: [external]
         environment:
@@ -1291,7 +1291,7 @@ suites:
             . "$TESTSLIB"/pkgdb.sh
             distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils
 
-    tests/nested/perf/:
+    tests/perf/nested/:
         summary: Performance and Load tests preparation suite
         backends: [google-nested, google-nested-dev, qemu-nested]
         environment:

tests/lib/nested.sh

@@ -1647,3 +1647,25 @@ nested_wait_for_device_initialized_change() {
         sleep "$wait"
     done
 }
+
+nested_check_spread_results() {
+    SPREAD_LOG=$1
+    if [ -z "$SPREAD_LOG" ]; then
+        return 1
+    fi
+
+    if grep -eq "Successful tasks:" "$SPREAD_LOG"; then
+        if grep -E "Failed (task|suite|project)" "$SPREAD_LOG"; then
+            return 1
+        fi
+        if ! grep -eq "Aborted tasks: 0" "$SPREAD_LOG"; then
+            return 1
+        fi
+
+        if [ "$EXIT_STATUS" = "0" ]; then
+            return 0
+        fi    
+    else
+        return 1
+    fi
+}

tests/perf/main/install-many-snaps-no-wait/task.yaml

@@ -0,0 +1,59 @@
+summary: Ensure the system properly handles installing many snaps with the --no-wait flag
+
+details: |
+    Install different snaps many times based on arch availability. In this
+    scenario all the snaps are installed in parallel and the test checks
+    the system can handel all the installs and ramains running.
+
+kill-timeout: 30m
+
+execute: |
+    if [ -z "$NUM_SNAPS" ]; then
+        NUM_SNAPS=20
+    fi
+
+    LETTERS="$(echo {a..z})"
+    INCLUDED=0
+    CHANNEL="stable"
+    SNAP_LIST=""
+
+    # shellcheck disable=SC2086
+    for letter in $LETTERS; do
+        if [ "$INCLUDED" = "$NUM_SNAPS" ]; then
+            echo "already $NUM_SNAPS ready to install, now check other features"
+            break
+        fi
+
+        snaps="$(snap find --narrow "$letter")"
+        SNAP_NAMES="$(echo "$snaps" | awk '{if($4~/-/){print $1}}' | tail -n+2)"
+        for SNAP in $SNAP_NAMES; do
+            # Get the info from latest/$CHANNEL
+            # shellcheck disable=SC2153
+            if ! CHANNEL_INFO="$(snap info --unicode=never "$SNAP" | grep " latest/$CHANNEL: ")"; then
+                echo "Snap $SNAP not found"
+                continue
+            fi
+            if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then
+                SNAP_LIST="$SNAP_LIST $SNAP"
+                INCLUDED=$(( INCLUDED + 1 ))
+            fi
+
+            if [ "$INCLUDED" = "$NUM_SNAPS" ]; then
+                echo "already $NUM_SNAPS included in the list"
+                break
+            fi
+        done
+    done
+
+    echo "Installing snaps: $SNAP_LIST"
+    # shellcheck disable=SC2086
+    for SNAP in $SNAP_LIST; do
+        snap install --no-wait "--$CHANNEL" "$SNAP"
+    done
+
+    while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do
+        sleep 1
+    done
+
+    test "$(snap changes | grep -cE ' Done.*Install \".*\" snap')" -ge "$NUM_SNAPS"
+    systemctl is-active snapd

tests/perf/install-many-snaps/task.yaml → tests/perf/main/install-many-snaps/task.yaml

@@ -12,7 +12,7 @@ execute: |
         NUM_SNAPS=100
     fi
 
-    LETTERS="a b c d e f g h i j k l m n o p q r s t u v w x y z"
+    LETTERS="$(echo {a..z})"
     INSTALLED=0
     CHANNEL='stable'
 
@@ -34,7 +34,7 @@ execute: |
             fi
             PARAMS=""
             if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then
-                snap install "$SNAP" "--$CHANNEL"
+                snap install --no-wait "$SNAP" "--$CHANNEL"
             elif echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*classic$"; then
                 if "$TESTSTOOLS"/snaps-state is-confinement-supported classic; then
                     PARAMS="--classic"
@@ -51,7 +51,7 @@ execute: |
                 continue
             fi
             
-            if snap install "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then
+            if snap install --no-wait "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then
                 INSTALLED=$(( INSTALLED + 1 ))
             else 
                 # this could cause the failure https://bugs.launchpad.net/snapstore-server/+bug/2049071
@@ -62,7 +62,11 @@ execute: |
                 echo "already $NUM_SNAPS installed, now check other features"
                 break
             fi
-        done        
+        done
+    done
+
+    while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do
+        sleep 1        
     done
 
     snap refresh

tests/perf/nested/install-many-snaps-no-wait/task.yaml

@@ -0,0 +1,29 @@
+summary: Ensure the system handles properly a installing many snaps with --no-wait parameter
+
+details: |
+    Install different snaps many times based on arch availability. In this
+    scenario all the snaps are installed in parallel and the test checks
+    the system can handle all the installs and remains running.
+
+
+systems: [ubuntu-18*, ubuntu-2*]
+
+environment:
+    TEST: install-many-snaps-no-wait
+    NUM_SNAPS: 35
+
+execute: |
+    # Get the nested system to use
+    NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)"
+
+    # Get spread
+    SPREAD="$(tests.nested download spread)"
+
+    # Run spread test
+    export SPREAD_EXTERNAL_ADDRESS=localhost:8022
+    export PERF_NUM_SNAPS="$NUM_SNAPS"
+    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log
+
+    #shellcheck source=tests/lib/nested.sh
+    . "$TESTSLIB/nested.sh"
+    nested_check_spread_results spread.log

tests/perf/nested/install-many-snaps/task.yaml

@@ -0,0 +1,29 @@
+summary: Ensure that commands run when their core provided interfaces are connected
+
+details: |
+    Install a test snap that plugs as many core provided interfaces as is 
+    possible and verify the command can run (ie, don't test the interface 
+    functionality itself). This will help catch things like AppArmor 
+    policy syntax errors, seccomp policy parsing, udev querying bugs, etc.
+
+systems: [ubuntu-18*, ubuntu-2*]
+
+environment:
+    TEST: install-many-snaps
+    NUM_SNAPS: 100
+
+execute: |
+    # Get the nested system to use
+    NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)"
+
+    # Get spread
+    SPREAD="$(tests.nested download spread)"
+
+    # Run spread test
+    export SPREAD_EXTERNAL_ADDRESS=localhost:8022
+    export PERF_NUM_SNAPS="$NUM_SNAPS"
+    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log
+
+    #shellcheck source=tests/lib/nested.sh
+    . "$TESTSLIB/nested.sh"
+    nested_check_spread_results spread.log

tests/nested/perf/interfaces-many/task.yaml → tests/perf/nested/interfaces-many/task.yaml

@@ -49,4 +49,8 @@ execute: |
     export SPREAD_EXTERNAL_ADDRESS=localhost:8022
     export PERF_CONNECTIONS_PERCENTAGE="$CONNECTIONS_PERCENTAGE"
     export PERF_DISCONNECT_INTERFACES="$DISCONNECT_INTERFACES"    
-    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST"
+    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log
+
+    #shellcheck source=tests/lib/nested.sh
+    . "$TESTSLIB/nested.sh"
+    nested_check_spread_results spread.log

tests/nested/perf/parallel-installs/task.yaml → tests/perf/nested/parallel-installs/task.yaml

@@ -32,4 +32,8 @@ execute: |
     export PERF_CPU_LOAD="$CPU_LOAD"
     export SPREAD_EXTERNAL_ADDRESS=localhost:8022
     export PERF_NUM_PARALLEL="$NUM_PARALLEL"
-    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST" | tee spread.log
+    "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log
+
+    #shellcheck source=tests/lib/nested.sh
+    . "$TESTSLIB/nested.sh"
+    nested_check_spread_results spread.log