add galaxy job radar visualization tool stats collection role and scripts

group_vars/maintenance.yml

@@ -288,7 +288,19 @@ telegraf_plugins_extra:
       - timeout = "60s"
       - data_format = "influx"
       - interval = "2m"
-
+  galaxy_job_radar_visualization_stats:
+    plugin: "exec"
+    config:
+      - commands = [
+        "{{ custom_telegraf_env }} /usr/bin/galaxy_failed_jobs_by_destination.sh",
+        "{{ custom_telegraf_env }} /usr/bin/galaxy_most_used_tools_by_destination.sh",
+        "{{ custom_telegraf_env }} /usr/bin/galaxy_num_anonymous_jobs_by_destination.sh",
+        "{{ custom_telegraf_env }} /usr/bin/galaxy_unique_users_job_count_by_destination.sh",
+        "{{ custom_telegraf_env }} /usr/bin/galaxy_longest_running_jobs_by_destination.sh",
+        ]
+      - timeout = "240s"
+      - data_format = "influx"
+      - interval = "1h"
 
 # Role: hxr.monitor-cluster
 monitor_condor: true

maintenance.yml

@@ -140,6 +140,7 @@
     # - usegalaxy-eu.fix-user-quotas
     - usegalaxy-eu.remove-orphan-condor-jobs
     - ssh_hardening
+    - usegalaxy-eu.job-radar-stats-influxdb
     - dj-wasabi.telegraf
     # - usegalaxy-eu.fix-stop-ITs
     - usegalaxy-eu.logrotate

roles/usegalaxy-eu.job-radar-stats-influxdb/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for galaxy-job-radar

roles/usegalaxy-eu.job-radar-stats-influxdb/files/galaxy_failed_jobs_by_destination.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Description: This script uses gxadmin to get the errored jobs from the Galaxy server in the last 1 hour and outputs the count of errored jobs by destination in InfluxDB line protocol format.
+
+gxadmin tsvquery errored-jobs 1 | awk '{count[$7]++} END {for (dest in count) printf "errored_jobs_by_destination,destination_id=%s,state=errored count=%d\n", dest, count[dest]}'

roles/usegalaxy-eu.job-radar-stats-influxdb/files/galaxy_longest_running_jobs_by_destination.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Description: This script uses gxadmin to get the longest running jobs on the Galaxy server and outputs the longest running jobs by destination in InfluxDB line protocol format.
+
+gxadmin query q "COPY (SELECT j.id, j.tool_id, j.destination_id, EXTRACT(EPOCH FROM (NOW() - jsh.running_since)) / 3600 AS hours_since_running FROM job j JOIN LATERAL (SELECT MIN(create_time) AS running_since FROM job_state_history jsh WHERE jsh.job_id = j.id AND jsh.state = 'running') jsh ON true WHERE j.state = 'running' ORDER BY hours_since_running DESC) TO STDOUT WITH (FORMAT CSV, HEADER FALSE, DELIMITER ',');" | awk -F, '{printf "longest_running_jobs,job_id=%s,tool_id=%s,destination_id=%s hours_since_running=%d\n", $1, $2, $3, $4}'

roles/usegalaxy-eu.job-radar-stats-influxdb/files/galaxy_most_used_tools_by_destination.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Description: This script uses gxadmin to count the most used tools on the Galaxy server and outputs the count of most used tools by destination in InfluxDB line protocol format.
+
+gxadmin query q "COPY (SELECT regexp_replace(tool_id, '/[^/]+$', '') AS tool_id_no_version, destination_id, COUNT(*) AS job_count FROM job WHERE create_time >= (now() AT TIME ZONE 'UTC' - INTERVAL '1 hours') AND tool_id ILIKE 'toolshed%' AND destination_id IS NOT NULL GROUP BY tool_id_no_version, destination_id ORDER BY job_count DESC) TO STDOUT WITH (FORMAT CSV, HEADER FALSE, DELIMITER ',');" | awk -F, '{printf "most_used_tools_by_destination,tool_id=%s,destination_id=%s job_count=%d\n", $1, $2, $3}'

roles/usegalaxy-eu.job-radar-stats-influxdb/files/galaxy_num_anonymous_jobs_by_destination.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Description: This script uses gxadmin to count the number of jobs run by anonymous users on the Galaxy server and outputs the count of anonymous users by destination in InfluxDB line protocol format.
+
+gxadmin csvquery queue-detail | awk -F',' '/Anonymous User/ {count[$9]++} END {for (id in count) print "anonymous_user_jobs_by_destination,destination_id=" id ", count=" count[id]}'

roles/usegalaxy-eu.job-radar-stats-influxdb/files/galaxy_unique_users_job_count_by_destination.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Description: This script uses gxadmin to count the unique users running jobs on the Galaxy server and outputs the count of unique users by destination in InfluxDB line protocol format.
+
+gxadmin csvquery queue-detail | grep running | awk -F, '{users[$9][$5]=1} END {for (dest in users) {count=0; for (user in users[dest]) count++; printf "num_unique_users_jobs_by_destination,destination_id=%s,state=running unique_user_count=%d\n", dest, count}}'

roles/usegalaxy-eu.job-radar-stats-influxdb/meta/main.yml

@@ -0,0 +1,54 @@
+galaxy_info:
+  author: your name
+  description: Galaxy stats collection scripts for galaxy job radar visualization tool
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags:
+    []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies:
+  []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

roles/usegalaxy-eu.job-radar-stats-influxdb/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- name: Copy the gxadmin bash scripts for collecting stats for galaxy job radar visualization tool
+  ansible.builtin.copy:
+    src: '{{ item }}'
+    dest: 'usr/bin/{{ item }}'
+    owner: 'root'
+    group: 'root'
+    mode: 0755
+  loop:
+    - 'galaxy_failed_jobs_by_destination.sh'
+    - 'galaxy_most_used_tools_by_destination.sh'
+    - 'galaxy_num_anonymous_jobs_by_destination.sh'
+    - 'galaxy_unique_users_job_count_by_destination.sh'
+    - 'galaxy_longest_running_jobs_by_destination.sh'