From f476b3eccd1a478d85abdcfce3aa9243f28281a3 Mon Sep 17 00:00:00 2001 From: Sergei Petrosian Date: Fri, 14 Feb 2025 13:48:36 +0100 Subject: [PATCH] ci: Add test plan that runs CI tests and customize it for each role * Calculate number of managed nodes with this formula: (( number_of_test_playbooks / 10 + 1 )) * Add README explaining how to run the plan locally and remotely Signed-off-by: Sergei Petrosian --- .codespellrc | 2 + .github/workflows/tft.yml | 4 +- plans/README-plans.md | 55 ++++++++++++++++++++++++ plans/test_playbooks_parallel.fmf | 69 +++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 plans/README-plans.md create mode 100644 plans/test_playbooks_parallel.fmf diff --git a/.codespellrc b/.codespellrc index 8f96f52..0e0654d 100644 --- a/.codespellrc +++ b/.codespellrc @@ -5,4 +5,6 @@ check-hidden = true ignore-multiline-regex = codespell:ignore-begin.*codespell:ignore-end ignore-words = .codespell_ignores # skip-file is not available https://github.com/codespell-project/codespell/pull/2759 +# .pandoc_template.html5 contains a typo in Licence that we shouldn't edit +# .README.html is generated from README.md automatically - no need to check spelling skip = .pandoc_template.html5,.README.html diff --git a/.github/workflows/tft.yml b/.github/workflows/tft.yml index 0934b62..9bb0938 100644 --- a/.github/workflows/tft.yml +++ b/.github/workflows/tft.yml @@ -157,7 +157,6 @@ jobs: uses: sclorg/testing-farm-as-github-action@v3 if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform) with: - git_url: https://github.com/linux-system-roles/tft-tests git_ref: main pipeline_settings: '{ "type": "tmt-multihost" }' environment_settings: '{ "provisioning": { "tags": { "BusinessUnit": "system_roles" } } }' @@ -180,8 +179,7 @@ jobs: tf_scope: private api_key: ${{ secrets.TF_API_KEY_RH }} update_pull_request_status: false - tmt_hardware: '{ "memory": ">= ${{ needs.prepare_vars.outputs.memory }} MB" }' - tmt_plan_filter: "tag:general,tlog" + tmt_plan_filter: "tag:playbooks_parallel,tlog" - name: Set final commit status uses: myrotvorets/set-commit-status-action@master diff --git a/plans/README-plans.md b/plans/README-plans.md new file mode 100644 index 0000000..ce43e62 --- /dev/null +++ b/plans/README-plans.md @@ -0,0 +1,55 @@ +# Introduction CI Testing Plans + +Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tft.yml](https://github.com/linux-system-roles/tlog/blob/main/.github/workflows/tft.yml) GitHub workflow. + +The `plans/test_playbooks_parallel.fmf` plan is a test plan that runs test playbooks in parallel on multiple managed nodes. +`plans/test_playbooks_parallel.fmf` is generated centrally from `https://github.com/linux-system-roles/.github/`. +The automation calculates the number of managed nodes to provision with this formula: + +```plain +number-of-test-playbooks / 10 + 1 +``` + +The `plans/test_playbooks_parallel.fmf` plan does the following steps: + +1. Provisions systems to be used as a control node and as managed nodes. +2. Does the required preparation on systems. +3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh). + +The [tft.yml](https://github.com/linux-system-roles/tlog/blob/main/.github/workflows/tft.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access. +This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm). + +## Running Tests + +You can run tests locally with the `tmt try` cli or remotely in Testing Farm. + +### Running Tests Locally + +1. Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html). +2. Change to the role repository directory. +3. Modify `plans/test_playbooks_parallel.fmf` to suit your requirements: + 1. Due to [issue #3138](https://github.com/teemtee/tmt/issues/3138), comment out all managed nodes except for one. + 2. Optionally modify environment variables to, e.g. run only specified test playbooks by modifying `SYSTEM_ROLES_ONLY_TESTS`. +4. Enter `tmt try -p plans/test_playbooks_parallel `. + This command identifies the `plans/test_playbooks_parallel.fmf` plan and provisions local VMs, a control node and a managed node. +5. `tmt try` is in development and does not identify tests from URL automatically, so after provisioning the machines, you must type `t`, `p`, `t` from the interactive prompt to identify tests, run preparation steps, and run the tests. + +### Running in Testing Farm + +1. Install `testing-farm` as described in [Installation](https://gitlab.com/testing-farm/cli/-/blob/main/README.adoc#user-content-installation). +2. Change to the role repository directory. +3. If you want to run tests with edits in your branch, you need to commit and push changes first to some branch. +4. You can uncomment "Inject your ssh public key to test systems" discover step in the plan if you want to troubleshoot tests by SSHing into test systems in Testing Farm. +5. Enter `testing-farm request`. + Edit to your needs. + + ```bash + $ TESTING_FARM_API_TOKEN= \ + testing-farm request --pipeline-type="tmt-multihost" \ + --plan-filter="tag:playbooks_parallel" \ + --git-url "https://github.com//tlog" \ + --git-ref "" \ + --compose CentOS-Stream-9 \ + -e "SYSTEM_ROLES_ONLY_TESTS=tests_default.yml" \ + --no-wait + ``` diff --git a/plans/test_playbooks_parallel.fmf b/plans/test_playbooks_parallel.fmf new file mode 100644 index 0000000..1d52437 --- /dev/null +++ b/plans/test_playbooks_parallel.fmf @@ -0,0 +1,69 @@ +summary: A general test for a system role +tag: playbooks_parallel +provision: + # TF uses `how: artemis`, and `tmt try`` uses `how: virtual`. + # Hence there is no need to define `how` explicitly. + - name: control-node1 + role: control_node + # `connection: system` is required for `how: virtual` to assign VMs a real + # IP making SSH configuration easier. + # This setting is ignored in `artemis`, so we can leave it as is. + connection: system + - name: managed-node1 + role: managed_node + connection: system +environment: + ANSIBLE_VER: 2.17 + REPO_NAME: tlog + PYTHON_VERSION: 3.12 + SYSTEM_ROLES_ONLY_TESTS: "" # tests_default.yml + TEST_LOCAL_CHANGES: true + PR_NUM: "" + LINUXSYSTEMROLES_USER: "" + LINUXSYSTEMROLES_DOMAIN: "" + LINUXSYSTEMROLES_SSH_KEY: "" + ARTIFACTS_DIR: "" + ARTIFACTS_URL: "" + LSR_TFT_DEBUG: false +prepare: + - name: Use vault.centos.org repos (CS 7, 8 EOL workaround) + script: | + if grep -q 'CentOS Stream release 8' /etc/redhat-release; then + sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo + fi + if grep -q 'CentOS Linux release 7.9' /etc/redhat-release; then + sed -i '/^mirror/d;s/#\?\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo + fi + # Replace with feature: epel: enabled once https://github.com/teemtee/tmt/pull/3128 is merged + - name: Enable epel to install beakerlib + script: | + # CS 10 and Fedora doesn't require epel + if grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then + exit 0 + fi + yum install epel-release yum-utils -y + yum-config-manager --enable epel epel-debuginfo epel-source +discover: + - name: Prepare managed node + how: fmf + where: managed_node + filter: tag:prep_managed_node + url: https://github.com/linux-system-roles/tft-tests + ref: main + - name: Run test playbooks from control_node + how: fmf + where: control_node + filter: tag:test_playbooks + url: https://github.com/linux-system-roles/tft-tests + ref: main + # Uncomment this step for troubleshooting + # This is required because currently testing-farm cli doesn't support running multi-node plans + # You can set ID_RSA_PUB in the environment section above to your public key to distribute it to nodes + # - name: Inject your ssh public key to test systems + # how: fmf + # where: control_node + # filter: tag:reserve_system + # url: https://github.com/linux-system-roles/tft-tests + # ref: main +execute: + how: tmt