tests: include new testing tools and utils (#11399)

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from dc6be12e76..575f5d2016

575f5d2016 Merge pull request #13 from snapcore/new-utils-section
4701612434 avoid installing and removing snapd in ubuntu focal
4d0317633d shellcheck in xenial is not needed
d95cb2e627 Moving some tools to the new utils section
839a0ce808 Merge pull request #10 from snapcore/new-spread-shellcheck-tool
5d5efd6fbe Merge pull request #12 from snapcore/run-shellchek-in-old-version
497b7d691f Force testing shellchek with versions 0.8 and 0.7
b3cb5a5a0a Update the copyright
872b64e05b Merge pull request #11 from Meulengracht/main
aa8799d0f8 tools/repack-kernel: fix SC2061
a44a987381 Merge pull request #9 from snapcore/check-spread-yaml-files
5cfa7cb84d New shellcheck for spread tasks
a9f6e09e0d Merge pull request #8 from Meulengracht/main
cd7926a85f New check-test-format tool
210b232d9c tools/repack-kernel: fix additional shellchecks
ac67796b5c Merge pull request #7 from Meulengracht/main
13fd7270b7 tools/repack-kernel: PR feedback comments
3844702a17 Merge pull request #6 from Meulengracht/main
4afdeefe42 tests/repack-kernel: move restore above execute to pass ./run-checks
e755eca0f8 Merge pull request #5 from Meulengracht/main
6199fc48f9 tools: add core22 targets in os.query
057919c07a Merge pull request #4 from mvo5/remove-ubuntu-21.04-eol
26b35af9b3 Merge pull request #1 from Meulengracht/main
b0fa82117a many: remove Ubuntu 21.04 (hirsute), it's EOL
878fd76976 repack-kernel: remove some cds and add a newline
4b84abd26b repack-kernel: fix issue in test related to wrong help message pattern
2e7fd675a7 repack-kernel: additional fixes to error messages. additional failure tests
545d72ef20 repack-kernel: additional fixes to tool based on feedback, additional tests
da1e43b4bb repack-kernel: change the install apt command for initramfs
131bcd11b2 repack-kernel: fix prepare command and error message having to much copy-pasta
9b2aef208f repack-kernel: add more tests, fix setup command
b98c024146 review feedback
5b76f3fe99 remove nested.sh, restructure repack-kernel and add test
0e5b5052e3 nested.sh: add missing command for waiting for snap command
9591a6e700 nested.sh: fix issues with OVMF
4db61f2b13 remove scripts that were hard to reuse, handle missing variables
9b956243e1 use env for path instead of hardcoding
b8e9d73349 core: add shared shell scripts for building image and controlling the L2 VM

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: 575f5d201619fc18f83c404222fc98e2ff696b04

* Usinr new utils and tools from the snapd-testing-tools project

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from 575f5d2016..61a4c3142d

61a4c3142d Merge pull request #14 from snapcore/include-support-s390x
3a1a65f6e1 Update the comments about shellchek
5ddfb7e1db New support for s390x arch in os.query tool

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: 61a4c3142d43d3e72330fd570ee16106d861ed24

.github/workflows/test.yaml

@@ -338,7 +338,7 @@ jobs:
       run: |
         if [ -e spread.log ]; then
             echo "Running spread log analyzer"
-            ./tests/lib/tools/log-parser spread.log --print-detail error-debug --output spread-results.json --cut 100
+            ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --print-detail error-debug --output spread-results.json --cut 100
             while IFS= read -r line; do
                 if [ ! -z "$line" ]; then
                     echo "Reporting spread error..."
@@ -420,7 +420,7 @@ jobs:
       run: |
         if [ -e spread.log ]; then
             echo "Running spread log analyzer"
-            ./tests/lib/tools/log-parser spread.log --print-detail error --output spread-results.json --cut 100
+            ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --print-detail error --output spread-results.json --cut 100
             while IFS= read -r line; do
                 if [ ! -z "$line" ]; then
                     echo "Reporting spread error..."

run-checks

@@ -205,7 +205,7 @@ if [ "$STATIC" = 1 ]; then
         fi
         unset regexp
         # also run spread-shellcheck
-        ./spread-shellcheck spread.yaml tests --exclude "$exclude_tools_path"
+        ./tests/lib/external/snapd-testing-tools/utils/spread-shellcheck spread.yaml tests --exclude "$exclude_tools_path"
     fi
 
     echo "Checking spelling errors"
@@ -267,7 +267,7 @@ if [ "$STATIC" = 1 ]; then
 
     if [ -z "${SKIP_TESTS_FORMAT_CHECK-}" ] || [ "$SKIP_TESTS_FORMAT_CHECK" = 0 ]; then
         echo "Checking tests formatting"
-        ./tests/utils/check-test-format ./tests
+        ./tests/lib/external/snapd-testing-tools/utils/check-test-format ./tests
     fi
 fi
 

tests/lib/external/snapd-testing-tools/.github/workflows/tests.yaml

@@ -16,10 +16,15 @@ jobs:
 
     - name: Run shellCheck for tools
       run: |
+          # Run shellcheck from the snap which most likely is newer than what the distro provides
           sudo apt-get remove --purge shellcheck
           sudo snap install shellcheck
+          find tools utils -type f -exec sh -c "head -n 1 {} | egrep -a 'bin/bash|bin/sh' >/dev/null" \; -print -exec shellcheck {} \;
 
-          find tools -type f -exec sh -c "head -n 1 {} | egrep -a 'bin/bash|bin/sh' >/dev/null" \; -print -exec shellcheck {} \;
+          # Run shellcheck from the distro
+          sudo snap remove shellcheck
+          sudo apt-get install -y shellcheck
+          find tools utils -type f -exec sh -c "head -n 1 {} | egrep -a 'bin/bash|bin/sh' >/dev/null" \; -print -exec shellcheck {} \;
 
   test:
     needs: [unit-tests]

tests/lib/external/snapd-testing-tools/README.md

@@ -37,6 +37,8 @@ and on [Facebook](https://www.facebook.com/snapcraftio).
 
 The tools included in this project are intended to be reused by other projects.
 
+Tools are supported in all the systems included in spread.yaml file.
+
 Read the following considerations before adding new tools:
 
  - Each tool needs to be accompanied by at least 1 spread test in `tests/<tool-name>/`
@@ -45,6 +47,23 @@ Read the following considerations before adding new tools:
  - All tools need to be as generic as possible
  - Each tool must also provide a command line interface (CLI), including _help_ output
 
+## Adding new utils
+
+The utils included in this project are intended to be reused by other projects.
+
+Utils are used as a complement for spread tests executions. Those are not intended to be used by spread tests.
+For example utils are used on github action workflows to analyze tests code and outputs.
+
+Utils are supported in ubuntu-18.04 and higher.
+
+Read the following considerations before adding new utils:
+
+ - Each util needs to be accompanied by at least 1 spread test in `tests/<util-name>/`
+ - At least 1 spread test needs to be included in the tests directory for each util
+ - If the util is a shell script, it needs to first pass a [ShellCheck](https://github.com/koalaman/shellcheck) assessment
+ - All utils need to be as generic as possible
+ - Each util must also provide a command line interface (CLI), including _help_ output
+
 
 ## Project status
 

tests/lib/external/snapd-testing-tools/spread.yaml

@@ -2,7 +2,7 @@ project: snapd-testing-tools
 
 environment:
     PROJECT_PATH: /root/snapd-testing-tools
-    PATH: $PATH:$PROJECT_PATH/tools
+    PATH: $PATH:$PROJECT_PATH/tools:$PROJECT_PATH/utils
 
 backends:
     google:

tests/lib/external/snapd-testing-tools/tests/os.query/task.yaml

@@ -9,6 +9,7 @@ execute: |
             os.query is-trusty
             os.query is-classic
             ! os.query is-core
+            ! os.query is-s390x
             ;;
         ubuntu-16.04-64)
             os.query is-xenial

tests/lib/external/snapd-testing-tools/tests/repack-kernel/task.yaml

@@ -3,14 +3,6 @@ summary: tests for the repack-kernel tool
 # ubuntu-core-initramfs seems to be only available on ubuntu-20.04
 systems: [ubuntu-20.04-64]
 
-prepare: |
-    # install dependencies
-    apt update -yqq
-    apt install snapd
-
-restore: |
-    apt remove snapd -yqq
-
 execute: |
     repack-kernel | MATCH 'usage: repack-kernel <command> <opts>'
     repack-kernel -h | MATCH 'usage: repack-kernel <command> <opts>'
@@ -47,7 +39,7 @@ execute: |
     repack-kernel pack "$kerneldir" --filename=pc-kernel.snap
     rm -rf "$kerneldir"
 
-    if ![ -f "pc-kernel.snap" ]; then
+    if ! [ -f "pc-kernel.snap" ]; then
         echo "Repack failed, pc-kernel.snap was not created"
         exit 1
     fi

tests/lib/external/snapd-testing-tools/tests/spread-shellcheck/task.yaml

@@ -0,0 +1,37 @@
+summary: smoke test for the spread-shellcheck tool
+
+systems: [ ubuntu-20.04-64 ]
+
+prepare: |
+    snap install shellcheck
+
+restore: |
+    snap remove shellcheck
+
+execute: |
+    spread-shellcheck -h | MATCH "usage: spread-shellcheck"
+
+    # Check the format of the spread tests in this project
+    spread-shellcheck "$PROJECT_PATH/tests"
+
+    # Check failing tasks
+    cp "$PWD/tasks/task1" "$PWD/tasks/task.yaml"
+    spread-shellcheck "$PWD/tasks" 2>&1 | MATCH "SC1035"
+    rm "$PWD/tasks/task.yaml"
+
+    cp "$PWD/tasks/task2" "$PWD/tasks/task.yaml"
+    spread-shellcheck "$PWD/tasks" 2>&1 | MATCH "SC1035"
+    rm "$PWD/tasks/task.yaml"
+
+    cp "$PWD/tasks/task3" "$PWD/tasks/task.yaml"
+    spread-shellcheck "$PWD/tasks" 2>&1 | MATCH "SC1035"
+    rm "$PWD/tasks/task.yaml"
+
+    cp "$PWD/tasks/task4" "$PWD/tasks/task.yaml"
+    spread-shellcheck "$PWD/tasks" 2>&1 | MATCH "SC1035"
+    rm "$PWD/tasks/task.yaml"
+
+    # Check that dirs can be excluded
+    cp "$PWD/tasks/task4" "$PWD/tasks/task.yaml"
+    spread-shellcheck "$PROJECT_PATH/tests" -e "$PWD/tasks"
+    rm "$PWD/tasks/task.yaml"

tests/lib/external/snapd-testing-tools/tests/spread-shellcheck/tasks/task1

@@ -0,0 +1,13 @@
+summary: this is the summary
+
+prepare: |
+    echo "preparing"
+
+restore: |
+    echo "restoring"
+
+debug: |
+    echo "debuging"
+
+execute: |
+    ![ -z "something" ]

tests/lib/external/snapd-testing-tools/tests/spread-shellcheck/tasks/task2

@@ -0,0 +1,13 @@
+summary: this is the summary
+
+prepare: |
+    ![ -z "something" ]
+
+restore: |
+    echo "restoring"
+
+debug: |
+    echo "debuging"
+
+execute: |
+    echo "executing"

tests/lib/external/snapd-testing-tools/tests/spread-shellcheck/tasks/task3

@@ -0,0 +1,13 @@
+summary: this is the summary
+
+prepare: |
+    echo "preparing"
+
+restore: |
+    ![ -z "something" ]
+
+debug: |
+    echo "debuging"
+
+execute: |
+    echo "executing"

tests/lib/external/snapd-testing-tools/tests/spread-shellcheck/tasks/task4

@@ -0,0 +1,13 @@
+summary: this is the summary
+
+prepare: |
+    echo "preparing"
+
+restore: |
+    echo "restoring"
+
+debug: |
+    ![ -z "something" ]
+
+execute: |
+    echo "executing"

tests/lib/external/snapd-testing-tools/tools/os.query

@@ -6,7 +6,7 @@ show_help() {
     echo "       os.query is-trusty, is-xenial, is-bionic, is-focal, is-impish"
     echo "       os.query is-ubuntu, is-debian, is-fedora, is-amazon-linux, is-arch-linux, is-centos, is-centos-7, is-centos-8, is-opensuse"
     echo "       os.query is-opensuse-tumbleweed, is-debian-sid, is-debian-11, is-debian-10"
-    echo "       os.query is-pc-amd64, is-pc-i386, is-arm, is-armhf, is-arm64"
+    echo "       os.query is-pc-amd64, is-pc-i386, is-arm, is-armhf, is-arm64, is-s390x"
     echo ""
     echo "Get general information about the current system"
 }
@@ -131,6 +131,10 @@ is_arm64() {
     uname -m | grep -Eq '(aarch64.*|armv8.*)'
 }
 
+is_s390x() {
+    uname -m | grep -qFx 's390x'
+}
+
 
 main() {
     if [ $# -eq 0 ]; then

tests/utils/check-test-format → tests/lib/external/snapd-testing-tools/utils/check-test-format

@@ -1,5 +1,11 @@
 #!/usr/bin/python3
 
+"""
+This tool is used to varify a correct format of spread tests
+The imput is a directory which is scanned recursively and all
+the task.yaml files are check
+"""
+
 import argparse
 import glob
 import os
@@ -69,8 +75,12 @@ def check_task_format(filepath):
         return False
 
     filemap = dict()
-    with open(filepath, "r") as task:
-        filemap = yaml.load(task, Loader=yamlordereddictloader.Loader)
+    try:
+        with open(filepath, "r") as task:
+            filemap = yaml.load(task, Loader=yamlordereddictloader.Loader)
+    except yaml.scanner.ScannerError:
+        print("Invalid task format, checks failed for task {}".format(filepath))
+        return False
 
     findings = check_keys_order(filemap.keys())
     findings.extend(check_mandatory_keys(filemap.keys()))

spread-shellcheck → tests/lib/external/snapd-testing-tools/utils/spread-shellcheck

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2018 Canonical Ltd
+# Copyright (C) 2022 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -14,22 +14,21 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
+import binascii
+import hashlib
+import itertools
 import logging
 import os
 import subprocess
-import argparse
-import itertools
-import hashlib
-import binascii
+import yaml
+
+from collections import namedtuple
 from concurrent.futures import ThreadPoolExecutor
-from threading import Lock
 from multiprocessing import cpu_count
-from typing import Dict
 from pathlib import Path
-from collections import namedtuple
-
-
-import yaml
+from threading import Lock
+from typing import Dict
 
 
 # default shell for shellcheck