From 30ad3d5b68bd07e50c30b376c798a5e882a44bd7 Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 23 Jul 2024 14:23:08 +0200 Subject: [PATCH 1/3] openqa-advanced-retrigger-jobs: Add usage+help --- openqa-advanced-retrigger-jobs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/openqa-advanced-retrigger-jobs b/openqa-advanced-retrigger-jobs index eeee6a6f..0bd55863 100755 --- a/openqa-advanced-retrigger-jobs +++ b/openqa-advanced-retrigger-jobs @@ -8,6 +8,40 @@ result="${result:-"result='incomplete'"}" additional_filters="${additional_filters+" and $additional_filters"}" comment="${comment:-""}" dry_run="${dry_run:-"0"}" + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Retrigger openQA jobs based on database queries. + +By default retriggers openQA jobs with result '$result' since '$failed_since' +on '$host'. + +Can be restricted to jobs that ran on worker by setting the variable 'WORKER' +and optionally 'INSTANCE'. + +Needs SSH access to the target openQA host '$host'. + +Options: + -h, --help display this help +EOF + exit "$1" +} + +opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1 +eval set -- "$opts" +while true; do + case "$1" in + -h | --help) usage 0 ;; + --) + shift + break + ;; + *) break ;; + esac +done + [ "$dry_run" = "1" ] && client_prefix="echo" # shellcheck disable=SC2029 for i in $(ssh "$host" "sudo -u geekotest psql --no-align --tuples-only --command=\"select id from jobs where (${worker_string}${result} and clone_id is null and t_finished >= '$failed_since'$additional_filters);\" openqa"); do From 56d668defe5f77f4c628ac9bed820df65f326c8e Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 23 Jul 2024 14:43:15 +0200 Subject: [PATCH 2/3] openqa-advanced-retrigger-jobs: Allow to import for testability and extendability --- openqa-advanced-retrigger-jobs | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/openqa-advanced-retrigger-jobs b/openqa-advanced-retrigger-jobs index 0bd55863..a81f071d 100755 --- a/openqa-advanced-retrigger-jobs +++ b/openqa-advanced-retrigger-jobs @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/bin/bash -e #worker="${worker:-"openqaworker4"}" host="${host:-"openqa.opensuse.org"}" failed_since="${failed_since:-"$(date -I)"}" @@ -29,22 +29,26 @@ EOF exit "$1" } -opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1 -eval set -- "$opts" -while true; do - case "$1" in - -h | --help) usage 0 ;; - --) - shift - break - ;; - *) break ;; - esac -done - -[ "$dry_run" = "1" ] && client_prefix="echo" -# shellcheck disable=SC2029 -for i in $(ssh "$host" "sudo -u geekotest psql --no-align --tuples-only --command=\"select id from jobs where (${worker_string}${result} and clone_id is null and t_finished >= '$failed_since'$additional_filters);\" openqa"); do - $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/restart - [ -n "$comment" ] && $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/comments text="$comment" -done +main() { + opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1 + eval set -- "$opts" + while true; do + case "$1" in + -h | --help) usage 0 ;; + --) + shift + break + ;; + *) break ;; + esac + done + + [ "$dry_run" = "1" ] && client_prefix="echo" + # shellcheck disable=SC2029 + for i in $(ssh "$host" "sudo -u geekotest psql --no-align --tuples-only --command=\"select id from jobs where (${worker_string}${result} and clone_id is null and t_finished >= '$failed_since'$additional_filters);\" openqa"); do + $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/restart + [ -n "$comment" ] && $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/comments text="$comment" + done +} + +caller 0 > /dev/null || main "$@" From 00ab1782411e2e2e17f72a1f4c371e06918c902b Mon Sep 17 00:00:00 2001 From: Oliver Kurz Date: Tue, 23 Jul 2024 17:38:10 +0200 Subject: [PATCH 3/3] Rewrite openqa-advanced-retrigger-jobs in python --- openqa-advanced-retrigger-jobs | 146 +++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 51 deletions(-) diff --git a/openqa-advanced-retrigger-jobs b/openqa-advanced-retrigger-jobs index a81f071d..d56561ec 100755 --- a/openqa-advanced-retrigger-jobs +++ b/openqa-advanced-retrigger-jobs @@ -1,54 +1,98 @@ -#!/bin/bash -e -#worker="${worker:-"openqaworker4"}" -host="${host:-"openqa.opensuse.org"}" -failed_since="${failed_since:-"$(date -I)"}" -instance_string="${INSTANCE+" and instance='$INSTANCE'"}" -worker_string="${WORKER+"assigned_worker_id in (select id from workers where (host='$WORKER'$instance_string)) and "}" -result="${result:-"result='incomplete'"}" -additional_filters="${additional_filters+" and $additional_filters"}" -comment="${comment:-""}" -dry_run="${dry_run:-"0"}" - -usage() { - cat << EOF -Usage: $0 [OPTIONS] +#!/usr/bin/env python3 +""" Retrigger openQA jobs based on database queries. -By default retriggers openQA jobs with result '$result' since '$failed_since' -on '$host'. - -Can be restricted to jobs that ran on worker by setting the variable 'WORKER' -and optionally 'INSTANCE'. - -Needs SSH access to the target openQA host '$host'. - -Options: - -h, --help display this help -EOF - exit "$1" -} - -main() { - opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1 - eval set -- "$opts" - while true; do - case "$1" in - -h | --help) usage 0 ;; - --) - shift - break - ;; - *) break ;; - esac - done - - [ "$dry_run" = "1" ] && client_prefix="echo" - # shellcheck disable=SC2029 - for i in $(ssh "$host" "sudo -u geekotest psql --no-align --tuples-only --command=\"select id from jobs where (${worker_string}${result} and clone_id is null and t_finished >= '$failed_since'$additional_filters);\" openqa"); do - $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/restart - [ -n "$comment" ] && $client_prefix openqa-cli api --host "$host" -X POST jobs/"$i"/comments text="$comment" - done -} - -caller 0 > /dev/null || main "$@" +Needs SSH access to the specified target openQA host. + +Simple example call retriggering all recent incompletes on the default host: + %(prog)s + +Advanced example retriggering failed instead of incompletes, verbose output, with custom starting date, +custom host and excluding jobs with \":investigate:\" in the name, executed as dry-run: + %(prog)s -vvvv --host openqa.example.org --failed-since '2000-01-01T10:00' --result failed \ +--additional-filters \"test not like '%%:investigate:%%'\" --dry-run +""" + +import argparse +import logging +import subprocess +import sys +from datetime import datetime + +logging.basicConfig() +log = logging.getLogger(sys.argv[0] if __name__ == "__main__" else __name__) + + +class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + """Preserve multi-line __doc__ and provide default arguments in help strings.""" + + pass + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=CustomFormatter) + parser.add_argument( + "-v", + "--verbose", + help="Increase verbosity level, specify multiple times to increase verbosity", + action="count", + default=1, + ) + parser.add_argument("-H", "--host", default="openqa.opensuse.org", help="Target openQA host") + parser.add_argument( + "-s", + "--failed-since", + default=datetime.today().isoformat(), + help="Filter jobs failed since this date", + ) + parser.add_argument("-w", "--worker", default=None, help="Filter jobs assigned to this worker") + parser.add_argument("-i", "--instance", default=None, help="Instance of the worker") + parser.add_argument("-r", "--result", default="incomplete", help="Filter jobs with this result") + parser.add_argument( + "-a", + "--additional-filters", + default=None, + help="Additional filters for the SQL query", + ) + parser.add_argument("-c", "--comment", default=None, help="Comment to add to the retriggered jobs") + parser.add_argument( + "-d", + "--dry-run", + action="store_true", + help="If set, only print the actions without executing", + ) + args = parser.parse_args() + logging_level = (5 - min(args.verbose, 4)) * 10 + log.setLevel(logging_level) + return args + +def post(dry_run: bool, host: str, route: str, *args): + cmd=('openqa-cli', 'api', '--host', host, '-X', 'POST', route, *args) + if dry_run: + return print(f"dry run: {cmd}") + subprocess.run(cmd, check=True) + +def main(args): + log.debug(args) + instance_string = f" and instance='{args.instance}'" if args.instance else "" + worker_string = f"assigned_worker_id in (select id from workers where (host='{args.worker}'{instance_string})) and " if args.worker else "" + additional_filters = f" and {args.additional_filters}" if args.additional_filters else "" + + query = ( + f"select id from jobs where ({worker_string}result='{args.result}' " + f"and clone_id is null and t_finished >= '{args.failed_since}'{additional_filters});" + ) + + log.debug(f"Using SQL query: '{query}' on {args.host}") + ssh_command = ('ssh', args.host, f"sudo -u geekotest psql --no-align --tuples-only --command=\"{query}\" openqa") + job_ids = subprocess.check_output(ssh_command).decode().splitlines() + + for job_id in job_ids: + post(args.dry_run, args.host, f"jobs/{job_id}/restart") + if args.comment: + post(args.dry_run, args.host, f"jobs/{job_id}/comments", f'text={args.comment}') + + +if __name__ == "__main__": + main(parse_args())