diff --git a/spread.yaml b/spread.yaml index 32d847cae08..21726413fe9 100644 --- a/spread.yaml +++ b/spread.yaml @@ -1069,7 +1069,7 @@ suites: restore: | "$TESTSLIB"/prepare-restore.sh --restore-suite - tests/perf/: + tests/perf/main/: summary: Performance and Load tests backends: [external] environment: @@ -1291,7 +1291,7 @@ suites: . "$TESTSLIB"/pkgdb.sh distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils - tests/nested/perf/: + tests/perf/nested/: summary: Performance and Load tests preparation suite backends: [google-nested, google-nested-dev, qemu-nested] environment: diff --git a/tests/lib/nested.sh b/tests/lib/nested.sh index 25354d1c680..85a519eb734 100755 --- a/tests/lib/nested.sh +++ b/tests/lib/nested.sh @@ -1647,3 +1647,25 @@ nested_wait_for_device_initialized_change() { sleep "$wait" done } + +nested_check_spread_results() { + SPREAD_LOG=$1 + if [ -z "$SPREAD_LOG" ]; then + return 1 + fi + + if grep -eq "Successful tasks:" "$SPREAD_LOG"; then + if grep -E "Failed (task|suite|project)" "$SPREAD_LOG"; then + return 1 + fi + if ! grep -eq "Aborted tasks: 0" "$SPREAD_LOG"; then + return 1 + fi + + if [ "$EXIT_STATUS" = "0" ]; then + return 0 + fi + else + return 1 + fi +} diff --git a/tests/perf/main/install-many-snaps-no-wait/task.yaml b/tests/perf/main/install-many-snaps-no-wait/task.yaml new file mode 100644 index 00000000000..deb450d33a9 --- /dev/null +++ b/tests/perf/main/install-many-snaps-no-wait/task.yaml @@ -0,0 +1,59 @@ +summary: Ensure the system properly handles installing many snaps with the --no-wait flag + +details: | + Install different snaps many times based on arch availability. In this + scenario all the snaps are installed in parallel and the test checks + the system can handel all the installs and ramains running. + +kill-timeout: 30m + +execute: | + if [ -z "$NUM_SNAPS" ]; then + NUM_SNAPS=20 + fi + + LETTERS="$(echo {a..z})" + INCLUDED=0 + CHANNEL="stable" + SNAP_LIST="" + + # shellcheck disable=SC2086 + for letter in $LETTERS; do + if [ "$INCLUDED" = "$NUM_SNAPS" ]; then + echo "already $NUM_SNAPS ready to install, now check other features" + break + fi + + snaps="$(snap find --narrow "$letter")" + SNAP_NAMES="$(echo "$snaps" | awk '{if($4~/-/){print $1}}' | tail -n+2)" + for SNAP in $SNAP_NAMES; do + # Get the info from latest/$CHANNEL + # shellcheck disable=SC2153 + if ! CHANNEL_INFO="$(snap info --unicode=never "$SNAP" | grep " latest/$CHANNEL: ")"; then + echo "Snap $SNAP not found" + continue + fi + if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then + SNAP_LIST="$SNAP_LIST $SNAP" + INCLUDED=$(( INCLUDED + 1 )) + fi + + if [ "$INCLUDED" = "$NUM_SNAPS" ]; then + echo "already $NUM_SNAPS included in the list" + break + fi + done + done + + echo "Installing snaps: $SNAP_LIST" + # shellcheck disable=SC2086 + for SNAP in $SNAP_LIST; do + snap install --no-wait "--$CHANNEL" "$SNAP" + done + + while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do + sleep 1 + done + + test "$(snap changes | grep -cE ' Done.*Install \".*\" snap')" -ge "$NUM_SNAPS" + systemctl is-active snapd diff --git a/tests/perf/install-many-snaps/task.yaml b/tests/perf/main/install-many-snaps/task.yaml similarity index 88% rename from tests/perf/install-many-snaps/task.yaml rename to tests/perf/main/install-many-snaps/task.yaml index 3375bcfe12a..f939b1fa441 100644 --- a/tests/perf/install-many-snaps/task.yaml +++ b/tests/perf/main/install-many-snaps/task.yaml @@ -12,7 +12,7 @@ execute: | NUM_SNAPS=100 fi - LETTERS="a b c d e f g h i j k l m n o p q r s t u v w x y z" + LETTERS="$(echo {a..z})" INSTALLED=0 CHANNEL='stable' @@ -34,7 +34,7 @@ execute: | fi PARAMS="" if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then - snap install "$SNAP" "--$CHANNEL" + snap install --no-wait "$SNAP" "--$CHANNEL" elif echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*classic$"; then if "$TESTSTOOLS"/snaps-state is-confinement-supported classic; then PARAMS="--classic" @@ -51,7 +51,7 @@ execute: | continue fi - if snap install "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then + if snap install --no-wait "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then INSTALLED=$(( INSTALLED + 1 )) else # this could cause the failure https://bugs.launchpad.net/snapstore-server/+bug/2049071 @@ -62,7 +62,11 @@ execute: | echo "already $NUM_SNAPS installed, now check other features" break fi - done + done + done + + while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do + sleep 1 done snap refresh diff --git a/tests/perf/interfaces-core-provided/task.yaml b/tests/perf/main/interfaces-core-provided/task.yaml similarity index 100% rename from tests/perf/interfaces-core-provided/task.yaml rename to tests/perf/main/interfaces-core-provided/task.yaml diff --git a/tests/perf/interfaces-snap-provided/task.yaml b/tests/perf/main/interfaces-snap-provided/task.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/task.yaml rename to tests/perf/main/interfaces-snap-provided/task.yaml diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml diff --git a/tests/perf/parallel-installs/task.yaml b/tests/perf/main/parallel-installs/task.yaml similarity index 100% rename from tests/perf/parallel-installs/task.yaml rename to tests/perf/main/parallel-installs/task.yaml diff --git a/tests/perf/nested/install-many-snaps-no-wait/task.yaml b/tests/perf/nested/install-many-snaps-no-wait/task.yaml new file mode 100644 index 00000000000..e09289c3656 --- /dev/null +++ b/tests/perf/nested/install-many-snaps-no-wait/task.yaml @@ -0,0 +1,29 @@ +summary: Ensure the system handles properly a installing many snaps with --no-wait parameter + +details: | + Install different snaps many times based on arch availability. In this + scenario all the snaps are installed in parallel and the test checks + the system can handle all the installs and remains running. + + +systems: [ubuntu-18*, ubuntu-2*] + +environment: + TEST: install-many-snaps-no-wait + NUM_SNAPS: 35 + +execute: | + # Get the nested system to use + NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)" + + # Get spread + SPREAD="$(tests.nested download spread)" + + # Run spread test + export SPREAD_EXTERNAL_ADDRESS=localhost:8022 + export PERF_NUM_SNAPS="$NUM_SNAPS" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/perf/nested/install-many-snaps/task.yaml b/tests/perf/nested/install-many-snaps/task.yaml new file mode 100644 index 00000000000..d67efdf927a --- /dev/null +++ b/tests/perf/nested/install-many-snaps/task.yaml @@ -0,0 +1,29 @@ +summary: Ensure that commands run when their core provided interfaces are connected + +details: | + Install a test snap that plugs as many core provided interfaces as is + possible and verify the command can run (ie, don't test the interface + functionality itself). This will help catch things like AppArmor + policy syntax errors, seccomp policy parsing, udev querying bugs, etc. + +systems: [ubuntu-18*, ubuntu-2*] + +environment: + TEST: install-many-snaps + NUM_SNAPS: 100 + +execute: | + # Get the nested system to use + NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)" + + # Get spread + SPREAD="$(tests.nested download spread)" + + # Run spread test + export SPREAD_EXTERNAL_ADDRESS=localhost:8022 + export PERF_NUM_SNAPS="$NUM_SNAPS" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/nested/perf/interfaces-many/task.yaml b/tests/perf/nested/interfaces-many/task.yaml similarity index 90% rename from tests/nested/perf/interfaces-many/task.yaml rename to tests/perf/nested/interfaces-many/task.yaml index 389d9a2fb75..521e049c5d6 100644 --- a/tests/nested/perf/interfaces-many/task.yaml +++ b/tests/perf/nested/interfaces-many/task.yaml @@ -49,4 +49,8 @@ execute: | export SPREAD_EXTERNAL_ADDRESS=localhost:8022 export PERF_CONNECTIONS_PERCENTAGE="$CONNECTIONS_PERCENTAGE" export PERF_DISCONNECT_INTERFACES="$DISCONNECT_INTERFACES" - "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/nested/perf/parallel-installs/task.yaml b/tests/perf/nested/parallel-installs/task.yaml similarity index 83% rename from tests/nested/perf/parallel-installs/task.yaml rename to tests/perf/nested/parallel-installs/task.yaml index 67929adc8fc..03ed8b3b353 100644 --- a/tests/nested/perf/parallel-installs/task.yaml +++ b/tests/perf/nested/parallel-installs/task.yaml @@ -32,4 +32,8 @@ execute: | export PERF_CPU_LOAD="$CPU_LOAD" export SPREAD_EXTERNAL_ADDRESS=localhost:8022 export PERF_NUM_PARALLEL="$NUM_PARALLEL" - "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST" | tee spread.log + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log