From 9134ef8daa86058e9ab618828add716a98fe7035 Mon Sep 17 00:00:00 2001 From: Sergio Cazzolato Date: Wed, 3 Apr 2024 10:01:18 -0300 Subject: [PATCH] tests: re-organize the perf test suite (#13672) * tests:new perf test to install many snaps with --no-wait and check changes Install different snaps many times based on arch availability. In this scenario all the snaps are installed in parallel and the test checks the system can handel all the installs and remains active. * performance tests reorganization To avoid executing perf tests when the nested tests are executed, the tests have been moved under tests/perf tests/perf/main -> suite with the tests executed in the target device tests/perf/nested -> suite with the tests executed in a nested environment * Update tests/perf/nested/install-many-snaps-no-wait/task.yaml Co-authored-by: Miguel Pires * Update tests/perf/nested/install-many-snaps-no-wait/task.yaml Co-authored-by: Miguel Pires * Update tests/perf/main/install-many-snaps-no-wait/task.yaml Co-authored-by: Miguel Pires * improve logs check and fixes based on review comments * addressing review comments --------- Co-authored-by: Miguel Pires --- spread.yaml | 4 +- tests/lib/nested.sh | 22 +++++++ .../main/install-many-snaps-no-wait/task.yaml | 59 +++++++++++++++++++ .../{ => main}/install-many-snaps/task.yaml | 12 ++-- .../interfaces-core-provided/task.yaml | 0 .../interfaces-snap-provided/task.yaml | 0 .../bin/run | 0 .../meta/snap.yaml | 0 .../bin/run | 0 .../meta/snap.yaml | 0 .../{ => main}/parallel-installs/task.yaml | 0 .../install-many-snaps-no-wait/task.yaml | 29 +++++++++ .../perf/nested/install-many-snaps/task.yaml | 29 +++++++++ .../nested}/interfaces-many/task.yaml | 6 +- .../nested}/parallel-installs/task.yaml | 6 +- 15 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 tests/perf/main/install-many-snaps-no-wait/task.yaml rename tests/perf/{ => main}/install-many-snaps/task.yaml (88%) rename tests/perf/{ => main}/interfaces-core-provided/task.yaml (100%) rename tests/perf/{ => main}/interfaces-snap-provided/task.yaml (100%) rename tests/perf/{ => main}/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run (100%) rename tests/perf/{ => main}/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml (100%) rename tests/perf/{ => main}/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run (100%) rename tests/perf/{ => main}/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml (100%) rename tests/perf/{ => main}/parallel-installs/task.yaml (100%) create mode 100644 tests/perf/nested/install-many-snaps-no-wait/task.yaml create mode 100644 tests/perf/nested/install-many-snaps/task.yaml rename tests/{nested/perf => perf/nested}/interfaces-many/task.yaml (90%) rename tests/{nested/perf => perf/nested}/parallel-installs/task.yaml (83%) diff --git a/spread.yaml b/spread.yaml index 32d847cae08..21726413fe9 100644 --- a/spread.yaml +++ b/spread.yaml @@ -1069,7 +1069,7 @@ suites: restore: | "$TESTSLIB"/prepare-restore.sh --restore-suite - tests/perf/: + tests/perf/main/: summary: Performance and Load tests backends: [external] environment: @@ -1291,7 +1291,7 @@ suites: . "$TESTSLIB"/pkgdb.sh distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils - tests/nested/perf/: + tests/perf/nested/: summary: Performance and Load tests preparation suite backends: [google-nested, google-nested-dev, qemu-nested] environment: diff --git a/tests/lib/nested.sh b/tests/lib/nested.sh index 25354d1c680..85a519eb734 100755 --- a/tests/lib/nested.sh +++ b/tests/lib/nested.sh @@ -1647,3 +1647,25 @@ nested_wait_for_device_initialized_change() { sleep "$wait" done } + +nested_check_spread_results() { + SPREAD_LOG=$1 + if [ -z "$SPREAD_LOG" ]; then + return 1 + fi + + if grep -eq "Successful tasks:" "$SPREAD_LOG"; then + if grep -E "Failed (task|suite|project)" "$SPREAD_LOG"; then + return 1 + fi + if ! grep -eq "Aborted tasks: 0" "$SPREAD_LOG"; then + return 1 + fi + + if [ "$EXIT_STATUS" = "0" ]; then + return 0 + fi + else + return 1 + fi +} diff --git a/tests/perf/main/install-many-snaps-no-wait/task.yaml b/tests/perf/main/install-many-snaps-no-wait/task.yaml new file mode 100644 index 00000000000..deb450d33a9 --- /dev/null +++ b/tests/perf/main/install-many-snaps-no-wait/task.yaml @@ -0,0 +1,59 @@ +summary: Ensure the system properly handles installing many snaps with the --no-wait flag + +details: | + Install different snaps many times based on arch availability. In this + scenario all the snaps are installed in parallel and the test checks + the system can handel all the installs and ramains running. + +kill-timeout: 30m + +execute: | + if [ -z "$NUM_SNAPS" ]; then + NUM_SNAPS=20 + fi + + LETTERS="$(echo {a..z})" + INCLUDED=0 + CHANNEL="stable" + SNAP_LIST="" + + # shellcheck disable=SC2086 + for letter in $LETTERS; do + if [ "$INCLUDED" = "$NUM_SNAPS" ]; then + echo "already $NUM_SNAPS ready to install, now check other features" + break + fi + + snaps="$(snap find --narrow "$letter")" + SNAP_NAMES="$(echo "$snaps" | awk '{if($4~/-/){print $1}}' | tail -n+2)" + for SNAP in $SNAP_NAMES; do + # Get the info from latest/$CHANNEL + # shellcheck disable=SC2153 + if ! CHANNEL_INFO="$(snap info --unicode=never "$SNAP" | grep " latest/$CHANNEL: ")"; then + echo "Snap $SNAP not found" + continue + fi + if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then + SNAP_LIST="$SNAP_LIST $SNAP" + INCLUDED=$(( INCLUDED + 1 )) + fi + + if [ "$INCLUDED" = "$NUM_SNAPS" ]; then + echo "already $NUM_SNAPS included in the list" + break + fi + done + done + + echo "Installing snaps: $SNAP_LIST" + # shellcheck disable=SC2086 + for SNAP in $SNAP_LIST; do + snap install --no-wait "--$CHANNEL" "$SNAP" + done + + while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do + sleep 1 + done + + test "$(snap changes | grep -cE ' Done.*Install \".*\" snap')" -ge "$NUM_SNAPS" + systemctl is-active snapd diff --git a/tests/perf/install-many-snaps/task.yaml b/tests/perf/main/install-many-snaps/task.yaml similarity index 88% rename from tests/perf/install-many-snaps/task.yaml rename to tests/perf/main/install-many-snaps/task.yaml index 3375bcfe12a..f939b1fa441 100644 --- a/tests/perf/install-many-snaps/task.yaml +++ b/tests/perf/main/install-many-snaps/task.yaml @@ -12,7 +12,7 @@ execute: | NUM_SNAPS=100 fi - LETTERS="a b c d e f g h i j k l m n o p q r s t u v w x y z" + LETTERS="$(echo {a..z})" INSTALLED=0 CHANNEL='stable' @@ -34,7 +34,7 @@ execute: | fi PARAMS="" if echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*-$"; then - snap install "$SNAP" "--$CHANNEL" + snap install --no-wait "$SNAP" "--$CHANNEL" elif echo "$CHANNEL_INFO" | MATCH "$CHANNEL:.*classic$"; then if "$TESTSTOOLS"/snaps-state is-confinement-supported classic; then PARAMS="--classic" @@ -51,7 +51,7 @@ execute: | continue fi - if snap install "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then + if snap install --no-wait "$SNAP" --$CHANNEL $PARAMS 2> stderr.out; then INSTALLED=$(( INSTALLED + 1 )) else # this could cause the failure https://bugs.launchpad.net/snapstore-server/+bug/2049071 @@ -62,7 +62,11 @@ execute: | echo "already $NUM_SNAPS installed, now check other features" break fi - done + done + done + + while snap changes | MATCH " (Do |Doing ).*Install \".*\" snap"; do + sleep 1 done snap refresh diff --git a/tests/perf/interfaces-core-provided/task.yaml b/tests/perf/main/interfaces-core-provided/task.yaml similarity index 100% rename from tests/perf/interfaces-core-provided/task.yaml rename to tests/perf/main/interfaces-core-provided/task.yaml diff --git a/tests/perf/interfaces-snap-provided/task.yaml b/tests/perf/main/interfaces-snap-provided/task.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/task.yaml rename to tests/perf/main/interfaces-snap-provided/task.yaml diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/bin/run diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-classic/meta/snap.yaml diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/bin/run diff --git a/tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml b/tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml similarity index 100% rename from tests/perf/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml rename to tests/perf/main/interfaces-snap-provided/test-snapd-policy-app-provider-core/meta/snap.yaml diff --git a/tests/perf/parallel-installs/task.yaml b/tests/perf/main/parallel-installs/task.yaml similarity index 100% rename from tests/perf/parallel-installs/task.yaml rename to tests/perf/main/parallel-installs/task.yaml diff --git a/tests/perf/nested/install-many-snaps-no-wait/task.yaml b/tests/perf/nested/install-many-snaps-no-wait/task.yaml new file mode 100644 index 00000000000..e09289c3656 --- /dev/null +++ b/tests/perf/nested/install-many-snaps-no-wait/task.yaml @@ -0,0 +1,29 @@ +summary: Ensure the system handles properly a installing many snaps with --no-wait parameter + +details: | + Install different snaps many times based on arch availability. In this + scenario all the snaps are installed in parallel and the test checks + the system can handle all the installs and remains running. + + +systems: [ubuntu-18*, ubuntu-2*] + +environment: + TEST: install-many-snaps-no-wait + NUM_SNAPS: 35 + +execute: | + # Get the nested system to use + NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)" + + # Get spread + SPREAD="$(tests.nested download spread)" + + # Run spread test + export SPREAD_EXTERNAL_ADDRESS=localhost:8022 + export PERF_NUM_SNAPS="$NUM_SNAPS" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/perf/nested/install-many-snaps/task.yaml b/tests/perf/nested/install-many-snaps/task.yaml new file mode 100644 index 00000000000..d67efdf927a --- /dev/null +++ b/tests/perf/nested/install-many-snaps/task.yaml @@ -0,0 +1,29 @@ +summary: Ensure that commands run when their core provided interfaces are connected + +details: | + Install a test snap that plugs as many core provided interfaces as is + possible and verify the command can run (ie, don't test the interface + functionality itself). This will help catch things like AppArmor + policy syntax errors, seccomp policy parsing, udev querying bugs, etc. + +systems: [ubuntu-18*, ubuntu-2*] + +environment: + TEST: install-many-snaps + NUM_SNAPS: 100 + +execute: | + # Get the nested system to use + NESTED_SPREAD_SYSTEM="$(tests.nested nested-system)" + + # Get spread + SPREAD="$(tests.nested download spread)" + + # Run spread test + export SPREAD_EXTERNAL_ADDRESS=localhost:8022 + export PERF_NUM_SNAPS="$NUM_SNAPS" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/nested/perf/interfaces-many/task.yaml b/tests/perf/nested/interfaces-many/task.yaml similarity index 90% rename from tests/nested/perf/interfaces-many/task.yaml rename to tests/perf/nested/interfaces-many/task.yaml index 389d9a2fb75..521e049c5d6 100644 --- a/tests/nested/perf/interfaces-many/task.yaml +++ b/tests/perf/nested/interfaces-many/task.yaml @@ -49,4 +49,8 @@ execute: | export SPREAD_EXTERNAL_ADDRESS=localhost:8022 export PERF_CONNECTIONS_PERCENTAGE="$CONNECTIONS_PERCENTAGE" export PERF_DISCONNECT_INTERFACES="$DISCONNECT_INTERFACES" - "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST" + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log diff --git a/tests/nested/perf/parallel-installs/task.yaml b/tests/perf/nested/parallel-installs/task.yaml similarity index 83% rename from tests/nested/perf/parallel-installs/task.yaml rename to tests/perf/nested/parallel-installs/task.yaml index 67929adc8fc..03ed8b3b353 100644 --- a/tests/nested/perf/parallel-installs/task.yaml +++ b/tests/perf/nested/parallel-installs/task.yaml @@ -32,4 +32,8 @@ execute: | export PERF_CPU_LOAD="$CPU_LOAD" export SPREAD_EXTERNAL_ADDRESS=localhost:8022 export PERF_NUM_PARALLEL="$NUM_PARALLEL" - "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/"$TEST" | tee spread.log + "$SPREAD" external:"$NESTED_SPREAD_SYSTEM":tests/perf/main/"$TEST" |& tee spread.log + + #shellcheck source=tests/lib/nested.sh + . "$TESTSLIB/nested.sh" + nested_check_spread_results spread.log