From d6f30212bb025b506f0a657f912e07d5ad3212df Mon Sep 17 00:00:00 2001 From: Fabiano Rosas Date: Fri, 23 Feb 2018 15:05:12 +0000 Subject: [PATCH] Remove BVT code We're not running build verification tests anymore, so cleanup this code. --- README.md | 21 +- ansible/README.md | 9 - ansible/bvt-host.yaml | 30 --- ansible/hosts.ini | 3 - ansible/roles/avocado-repo/tasks/main.yaml | 10 - ansible/roles/avocado/defaults/main.yaml | 10 - ansible/roles/avocado/files/host-os-bvt.ini | 10 - ansible/roles/avocado/handlers/main.yaml | 11 - ansible/roles/avocado/meta/main.yaml | 5 - ansible/roles/avocado/tasks/main.yaml | 127 ---------- .../roles/avocado/templates/avocado.conf.j2 | 11 - ansible/vars-bvt.yaml | 46 ---- pipeline/daily/stages.groovy | 30 --- scripts/host-os-bvt.py | 226 ------------------ 14 files changed, 4 insertions(+), 545 deletions(-) delete mode 100644 ansible/bvt-host.yaml delete mode 100644 ansible/roles/avocado-repo/tasks/main.yaml delete mode 100644 ansible/roles/avocado/defaults/main.yaml delete mode 100644 ansible/roles/avocado/files/host-os-bvt.ini delete mode 100644 ansible/roles/avocado/handlers/main.yaml delete mode 100644 ansible/roles/avocado/meta/main.yaml delete mode 100644 ansible/roles/avocado/tasks/main.yaml delete mode 100644 ansible/roles/avocado/templates/avocado.conf.j2 delete mode 100644 ansible/vars-bvt.yaml delete mode 100755 scripts/host-os-bvt.py diff --git a/README.md b/README.md index 2724d64..af592d7 100644 --- a/README.md +++ b/README.md @@ -108,16 +108,15 @@ by Jenkins Ansible playbooks, so you do not need to do it manually. ### Automatically setup Jenkins master and slave(s) using Ansible playbooks -There are three Ansible playbooks: one that sets up a Jenkins master node, -and two that set up Jenkins slave nodes, either for building or executing -Build Verification Tests. Read the +There are two Ansible playbooks: one that sets up a Jenkins master node +and another one that sets up a Jenkins slave node. Read the [Ansible instructions](ansible/README.md) for details on how to execute the playbooks. -If you wish to have a single system hosting the entire Jenkins instance, the +If you wish to have a single system hosting the entire Jenkins instance, both playbooks can be executed in the same system. Execute first the Jenkins master playbook, stop Jenkins service (systemctl stop jenkins) and then execute the -other playbooks. +Jenkins slave playbook. Note: The Jenkins playbooks may fail due to network errors. If you see HTTP request errors, try executing them again. @@ -181,18 +180,6 @@ of the slave in IP_ADDRESS job parameter. The other job parameters values do not need to be modified. You should have already executed the Jenkins slave playbook(s) on those slaves. -##### Create BVT slave in Jenkins web UI - -The Build Verification Tests (BVT) use [Avocado](https://avocado-framework.github.io/) -to execute tests on virtual machines. The tests require passwordless access to -sudo, so it is recommended that a separate machine is used forthat purpose. It -may be a virtual machine, the tests will then run in a nested virtualized -guest. - -To create the slave node from the Jenkins UI, follow the same instructions -above, setting the IP_ADDRESS and, additionally, changing the NODE_LABEL to -"bvt_slave_label". - #### Create builds jobs When the credentials are configured, execute the seed job at diff --git a/ansible/README.md b/ansible/README.md index 3db1ff3..dfdc341 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -62,15 +62,6 @@ artifacts to the configured remote server. - Path to the SSH known keys for remote hosts, usually `~/.ssh/known_hosts`. The local file is just copied to remote target. -- `bvt-host.yaml` playbook - - - Paths to SSH private and public keys used to communicate between master - and slave nodes. - - Path to the SSH private key used to upload artifacts to the configured - remote server. - - Path to the SSH known keys for remote hosts, usually - `~/.ssh/known_hosts`. The local file is just copied to remote target. - Provide the data requested by the playbooks (e.g. Jenkins admin user name/password and SSH keys locations) and wait for automatic setup to finish. diff --git a/ansible/bvt-host.yaml b/ansible/bvt-host.yaml deleted file mode 100644 index c083268..0000000 --- a/ansible/bvt-host.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -- name: Setup host for Build Verification Tests - hosts: bvt-host - remote_user: root - roles: - - selinux - - time - - packages-jenkins-slave - - user - - ssh - - avocado - vars_files: - - vars-bvt.yaml - vars_prompt: - - name: "jenkins_private_ssh_key_file_path" - prompt: "Enter Jenkins private SSH key file path" - default: "~/.ssh/jenkins_id_rsa" - private: no - - name: "jenkins_public_ssh_key_file_path" - prompt: "Enter Jenkins public SSH key file path" - default: "~/.ssh/jenkins_id_rsa.pub" - private: no - - name: "upload_server_user_private_ssh_key_file_path" - prompt: "Enter upload server user's private SSH key file path" - default: "~/.ssh/open-power-host-os-builds-bot_id_rsa" - private: no - - name: "known_hosts_file_path" - prompt: "Enter path to file containing known keys for upload server host" - default: "~/.ssh/known_hosts" - private: no diff --git a/ansible/hosts.ini b/ansible/hosts.ini index 17dea9a..e424cb4 100644 --- a/ansible/hosts.ini +++ b/ansible/hosts.ini @@ -9,6 +9,3 @@ host-os-jenkins-slave02.example.com [baremetal-ctrl] host-os-jenkins-slave04.example.com - -[bvt-host] -host-os-bvt-host.example.com diff --git a/ansible/roles/avocado-repo/tasks/main.yaml b/ansible/roles/avocado-repo/tasks/main.yaml deleted file mode 100644 index 3e24cdf..0000000 --- a/ansible/roles/avocado-repo/tasks/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Setup avocado repository - get_url: - url: http://avocado-project.org/data/repos/avocado-el.repo - dest: /etc/yum.repos.d/avocado.repo - force: yes - owner: root - group: root - mode: 0644 - tags: - - setup diff --git a/ansible/roles/avocado/defaults/main.yaml b/ansible/roles/avocado/defaults/main.yaml deleted file mode 100644 index f0150e9..0000000 --- a/ansible/roles/avocado/defaults/main.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -distro_packages: - - attr - - avocado - - avocado-plugins-vt - - libvirt - - policycoreutils-python - - qemu-kvm - - virt-install - - wget diff --git a/ansible/roles/avocado/files/host-os-bvt.ini b/ansible/roles/avocado/files/host-os-bvt.ini deleted file mode 100644 index f3ca80a..0000000 --- a/ansible/roles/avocado/files/host-os-bvt.ini +++ /dev/null @@ -1,10 +0,0 @@ -[provider] -uri: https://github.com/open-power-host-os/bvt.git -branch: master -[generic] -subdir: generic - -# A bug in avocado-vt prevents the tests from being listed without this -# https://github.com/avocado-framework/avocado-vt/issues/1222 -[multi_host_migration] -subdir: generic diff --git a/ansible/roles/avocado/handlers/main.yaml b/ansible/roles/avocado/handlers/main.yaml deleted file mode 100644 index 52354bc..0000000 --- a/ansible/roles/avocado/handlers/main.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: vt-install - command: python setup.py install - args: - chdir: "{{avocado_vt_repo_dir}}" - -- name: vt-bootstrap - command: avocado vt-bootstrap --vt-type {{item}} --vt-no-downloads --yes-to-all - with_items: - - qemu - - libvirt diff --git a/ansible/roles/avocado/meta/main.yaml b/ansible/roles/avocado/meta/main.yaml deleted file mode 100644 index 3edea1c..0000000 --- a/ansible/roles/avocado/meta/main.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -dependencies: - - role: epel - - role: avocado-repo - - role: pkg_install diff --git a/ansible/roles/avocado/tasks/main.yaml b/ansible/roles/avocado/tasks/main.yaml deleted file mode 100644 index 19e944b..0000000 --- a/ansible/roles/avocado/tasks/main.yaml +++ /dev/null @@ -1,127 +0,0 @@ -- name: Make sure we have a 'wheel' group - group: - name: wheel - state: present - tags: - - setup - -- name: Allow 'wheel' group to have passwordless sudo - lineinfile: - dest: /etc/sudoers - state: present - regexp: '^%wheel' - line: '%wheel ALL=(ALL) NOPASSWD: ALL' - validate: 'visudo -cf %s' - tags: - - setup - -- name: Add sudoers users to wheel group - user: - name: "{{user_name}}" - groups: wheel - append: yes - state: present - createhome: yes - tags: - - setup - -- name: Configure SSH key to download from remote server - copy: - src={{upload_server_user_private_ssh_key_file_path}} - dest="{{user_home_dir}}/.ssh/upload_server_id_rsa" - owner={{user_name}} group={{user_name}} mode=0600 - tags: - - setup - -- name: Add known keys for remote hosts - copy: - src={{known_hosts_file_path}} dest="{{jenkins_home_dir}}/.ssh/known_hosts" - owner={{user_name}} group={{user_name}} - tags: - - setup - -# This is necessary to create virbr0 interface. You can create it -# manually, but libvirt does the job for you. -- name: Start libvirtd service - service: - name: libvirtd - state: started - enabled: yes - tags: - - setup - -# Those ports are used to provide kickstart files via HTTP -- name: Open ports in firewalld - firewalld: - port: "{{avocado_http_ports}}" - state: enabled - permanent: yes - immediate: yes - tags: - - setup - -- name: Creating avocado directories - file: - path: "{{item}}" - state: directory - owner: root - group: root - mode: 0755 - with_items: "{{avocado_dirs}}" - tags: - - setup - -- name: Clone avocado-vt - git: - repo: "{{avocado_vt_repo}}" - version: "{{avocado_vt_branch}}" - dest: "{{avocado_vt_repo_dir}}" - update: yes - notify: - - vt-install - - vt-bootstrap - tags: - - setup - -- name: Setup avocado configuration - template: - src: avocado.conf.j2 - dest: "{{avocado_conf}}" - owner: root - group: root - mode: 0644 - notify: vt-bootstrap - tags: - - setup - -- name: Setup test providers - copy: - src: host-os-bvt.ini - dest: "{{test_providers_dir}}/host-os-bvt.ini" - owner: root - group: root - mode: 0644 - notify: vt-bootstrap - tags: - - setup - -- name: Cloning test providers - git: - repo: "{{item.repo}}" - dest: "{{item.dest}}" - update: yes - notify: vt-bootstrap - with_items: - - "{{avocado_repos}}" - tags: - - setup - -- name: Install BVT script - copy: - src: "{{playbook_dir}}/../scripts/host-os-bvt.py" - dest: /usr/bin/host-os-bvt - owner: root - group: root - mode: 0555 - tags: - - setup diff --git a/ansible/roles/avocado/templates/avocado.conf.j2 b/ansible/roles/avocado/templates/avocado.conf.j2 deleted file mode 100644 index a3be3c4..0000000 --- a/ansible/roles/avocado/templates/avocado.conf.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# {{avocado_conf}} - -[datadir.paths] -base_dir = {{avocado_dir}} -test_dir = {{avocado_test_dir}} -data_dir = {{avocado_data_dir}} -logs_dir = {{avocado_log_dir}} - -[sysinfo.collect] -enabled = True -profiler = True diff --git a/ansible/vars-bvt.yaml b/ansible/vars-bvt.yaml deleted file mode 100644 index 9fbff3b..0000000 --- a/ansible/vars-bvt.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -user_name: jenkins-slave -user_groups: '' -user_home_dir: "/home/{{user_name}}" -jenkins_home_dir: "{{user_home_dir}}" - -avocado_dir: "{{ansible_env.HOME}}/avocado" - -avocado_data_dir: "{{avocado_dir}}/data" -avocado_iso_dir: "{{avocado_dir}}/isos" -avocado_log_dir: "{{avocado_dir}}/job-results" -avocado_test_dir: "{{avocado_dir}}/tests" - -avocado_conf_dir: "{{ansible_env.HOME}}/.config/avocado" -avocado_conf: "{{avocado_conf_dir}}/avocado.conf" - -avocado_vt_repo: "https://github.com/open-power-host-os/avocado-vt.git" -avocado_vt_branch: "master" -avocado_vt_repo_dir: "{{ansible_env.HOME}}/bvt/avocado-vt" - -test_providers_dir: "{{avocado_data_dir}}/avocado-vt/test-providers.d" -test_providers_downloads_dir: "{{test_providers_dir}}/downloads" - -avocado_dirs: - - "{{ansible_env.HOME}}" - - "{{avocado_dir}}" - - "{{avocado_data_dir}}" - - "{{avocado_iso_dir}}" - - "{{avocado_log_dir}}" - - "{{avocado_test_dir}}" - - "{{avocado_conf_dir}}" - - "{{avocado_vt_repo_dir}}" - - "{{test_providers_dir}}" - - "{{test_providers_downloads_dir}}" - -avocado_repos: - - { repo: "https://github.com/autotest/tp-libvirt.git", - dest: "{{test_providers_downloads_dir}}/io-github-autotest-libvirt" } - - { repo: "https://github.com/autotest/tp-qemu.git", - dest: "{{test_providers_downloads_dir}}/io-github-autotest-qemu" } - - { repo: "https://github.com/spiceqa/tp-spice.git", - dest: "{{test_providers_downloads_dir}}/io-github-spiceqa-spice" } - - { repo: "https://github.com/open-power-host-os/bvt.git", - dest: "{{test_providers_downloads_dir}}/host-os-bvt" } - -avocado_http_ports: 8000-8004/tcp diff --git a/pipeline/daily/stages.groovy b/pipeline/daily/stages.groovy index e6ab217..64846e5 100644 --- a/pipeline/daily/stages.groovy +++ b/pipeline/daily/stages.groovy @@ -186,36 +186,6 @@ python host_os.py \ } } -def runBVT() { - String PREVIOUS_YUM_REPO_FILE_URL = - "${RSYNC_URL_PREFIX}$params.UPLOAD_SERVER_PERIODIC_BUILDS_DIR_PATH/" + - "latest/hostos.repo" - String CURRENT_YUM_REPO_FILE_URL = - "${RSYNC_URL_PREFIX}$params.UPLOAD_SERVER_BUILDS_DIR_PATH/" + - "$buildStages.buildTimestamp/hostos.repo" - - previousConfigParameter = '' - try { - utils.rsyncDownload(PREVIOUS_YUM_REPO_FILE_URL, 'previous_host_os.repo') - previousConfigParameter = '--previous-yum-config-file previous_host_os.repo' - } catch (hudson.AbortException exception) { - echo('Previous build not found, update test will be skipped.') - } - - try { - utils.rsyncDownload(CURRENT_YUM_REPO_FILE_URL, 'current_host_os.repo') - - sh """\ -sudo host-os-bvt \ - --current-yum-config-file current_host_os.repo \ - $previousConfigParameter \ -""" - } catch (Exception exception) { - echo('BVT execution failed') - currentBuild.result = 'UNSTABLE' - } -} - def commitToGitRepo() { String GITHUB_BOT_HTTP_URL = "ssh://git@github/$params.GITHUB_BOT_USER_NAME" diff --git a/scripts/host-os-bvt.py b/scripts/host-os-bvt.py deleted file mode 100755 index 8012fd4..0000000 --- a/scripts/host-os-bvt.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/env python2 - -import argparse -import logging -import shlex -import shutil -import subprocess - -AVOCADO_RUN_COMMAND = ( - "/usr/bin/avocado run --vt-type {vt_type} --vt-guest-os {vt_guest_os} " - "--failfast on {tests}") -# TODO: Parameterize those values -VT_TYPE = "libvirt" -VT_GUEST_OS = "CentOS.7.4" - -IMPORT_GUEST_TEST = ("io-github-autotest-qemu.unattended_install.import.import" - ".default_install.aio_native") -REMOVE_GUEST_TEST = "io-github-autotest-libvirt.remove_guest.without_disk" -INSTALL_BASE_OS_TEST = ("io-github-autotest-qemu.unattended_install.url.http_ks" - ".default_install.aio_native") -UPDATE_BASE_OS_TEST = "io-github-autotest-qemu.yum_update" -INSTALL_HOST_OS_TEST = "host-os-bvt.yum_install" -UPDATE_HOST_OS_TEST = "host-os-bvt.yum_update" - -# TODO: Parameterize those values -DISK_PATH = "/root/avocado/data/avocado-vt/images/centos74-ppc64le.qcow2" -DISK_BACKUP_PATH = DISK_PATH + ".backup" - -LOGGING_FORMAT = "%(message)s" -logging.basicConfig(level=logging.DEBUG, format=LOGGING_FORMAT) - - -def parse_cli_options(): - """ - Parse CLI options - - Returns: - Namespace: CLI options. Valid attributes: previous_yum_config_file, - current_yum_config_file. - """ - - parser = argparse.ArgumentParser() - parser.add_argument("--current-yum-config-file", - help="yum configuration file pointing to the current " - "Host OS repository", - default="host_os.repo") - parser.add_argument("--previous-yum-config-file", - help="yum configuration file pointing to a previous " - "Host OS repository", - default=None) - args = parser.parse_args() - return args - - -def get_vt_extra_params(parameter_values): - """ - Get a string to be appended to the 'avocado run' command setting - extra parameters for avocado-vt. - - Args: - parameter_values (dict): dictionary mapping parameter names to - their values - """ - vt_extra_params = " --vt-extra-params" - for name, value in parameter_values.items(): - vt_extra_params += " {name}={value}".format(**locals()) - return vt_extra_params - - -def execute_avocado_command(tests, vt_extra_parameter_values=None): - """ - Execute the tests using the Avocado framework. - - Args: - tests ([str]): name of the tests to be executed - vt_extra_parameter_values (dict): dictionary mapping avocado-vt - parameter names to their values - """ - cmd = AVOCADO_RUN_COMMAND.format( - vt_type=VT_TYPE, vt_guest_os=VT_GUEST_OS, tests=" ".join(tests)) - if vt_extra_parameter_values: - cmd += get_vt_extra_params(vt_extra_parameter_values) - - logging.debug("Executing: " + cmd) - subprocess.check_call(shlex.split(cmd), stderr=subprocess.STDOUT) - - -def update_guest(): - """ - Create a libvirt domain from an existing disk image and updates - the system using yum. - """ - logging.info("Updating an existing guest") - tests = [IMPORT_GUEST_TEST, UPDATE_BASE_OS_TEST] - extra_params = dict( - restore_image_after_testing=False, - yum_update_timeout=1800) - execute_avocado_command(tests, extra_params) - - -def install_guest(): - """ - Create a libvirt domain, installs the OS in an empty disk image and - updates the system using yum. - """ - logging.info("Installing the OS and updating a new guest") - tests = [INSTALL_BASE_OS_TEST, UPDATE_BASE_OS_TEST] - extra_params = dict( - restore_image_after_testing=False, - install_timeout=9000, - yum_update_timeout=1800) - execute_avocado_command(tests, extra_params) - - -def install_host_os(yum_config_file_path): - """ - Execute the test that installs the Host OS packages on top of a - working system. - - Args: - yum_config_file_path (str): path to a yum configuration file - pointing to a repository containing Host OS packages - """ - logging.info("Installing Host OS packages") - tests = [INSTALL_HOST_OS_TEST] - extra_params = dict( - restore_image_after_testing=False, - yum_install_timeout=1800, - yum_config_file_path=yum_config_file_path) - execute_avocado_command(tests, extra_params) - - -def update_host_os(yum_install_config_file_path, yum_update_config_file_path): - """ - Execute the test that installs the Host OS packages on top of a - working system and updates them to more recent version. - - Args: - yum_install_config_file_path (str): path to a yum configuration - file pointing to a repository containing outdated Host OS - packages - yum_update_config_file_path (str): path to a yum configuration - file pointing to a repository containing updated Host OS - packages - """ - logging.info("Updating a system with Host OS") - tests = [UPDATE_HOST_OS_TEST] - extra_params = dict( - restore_image_after_testing=False, - yum_install_timeout=1800, - yum_install_config_file_path=yum_install_config_file_path, - yum_update_timeout=1200, - yum_update_config_file_path=yum_update_config_file_path) - execute_avocado_command(tests, extra_params) - - -def remove_guest(): - """ - Remove a libvirt domain, leaving its disk image as is. - """ - logging.info("Removing the guest") - tests = [REMOVE_GUEST_TEST] - execute_avocado_command(tests) - - -def backup_disk_image(): - """ - Back up the disk image to a predefined path expected by avocado-vt. - """ - logging.info("Backing up disk image") - logging.debug("Copying from '{source}' to '{dest}'".format( - source=DISK_PATH, dest=DISK_BACKUP_PATH)) - shutil.copy(DISK_PATH, DISK_BACKUP_PATH) - - -def restore_disk_image(): - """ - Restore a previously backed up disk image. - """ - logging.info("Restoring disk backup") - logging.debug("Copying from '{source}' to '{dest}'".format( - source=DISK_BACKUP_PATH, dest=DISK_PATH)) - shutil.copy(DISK_BACKUP_PATH, DISK_PATH) - - -def execute_bvt(current_yum_config_file, previous_yum_config_file=None): - """ - Prepare a guest with the base OS and execute the Host OS install and - update tests on top of it. - - Args: - current_yum_config_file (str): path to a yum configuration - file pointing to a repository containing updated Host OS - packages - previous_yum_config_file (str): path to a yum configuration - file pointing to a repository containing outdated Host OS - packages - """ - try: - try: - update_guest() - except subprocess.CalledProcessError as error: - logging.warning(error) - try: - remove_guest() - except subprocess.CalledProcessError as remove_error: - logging.debug(remove_error) - install_guest() - backup_disk_image() - install_host_os(current_yum_config_file) - restore_disk_image() - if previous_yum_config_file: - update_host_os(previous_yum_config_file, current_yum_config_file) - restore_disk_image() - else: - logging.info( - "No previous repository provided, skipping update test") - except: - raise - finally: - remove_guest() - - -if __name__ == '__main__': - args = parse_cli_options() - execute_bvt(args.current_yum_config_file, args.previous_yum_config_file)