func: fix the last_log_index check and add a versions check

enos/modules/test_cluster_health/main.tf

@@ -10,7 +10,7 @@ terraform {
 }
 
 locals {
-  clean_token = trimspace(var.nomad_token) #Somewhere in the process, a newline is added to the token.
+  servers_addr = join(" ", var.servers)
 }
 
 resource "enos_local_exec" "run_tests" {
@@ -19,11 +19,12 @@ resource "enos_local_exec" "run_tests" {
     NOMAD_CACERT      = var.ca_file
     NOMAD_CLIENT_CERT = var.cert_file
     NOMAD_CLIENT_KEY  = var.key_file
-    NOMAD_TOKEN       = local.clean_token
+    NOMAD_TOKEN       = var.nomad_token
     SERVER_COUNT      = var.server_count
     CLIENT_COUNT      = var.client_count
     JOB_COUNT         = var.jobs_count
     ALLOC_COUNT       = var.alloc_count
+    SERVERS           = local.servers_addr
   }
 
   scripts = [
@@ -33,3 +34,20 @@ resource "enos_local_exec" "run_tests" {
     abspath("${path.module}/scripts/allocs.sh")
   ]
 }
+
+ resource "enos_local_exec" "verify_versions" {
+  environment = {
+    NOMAD_ADDR        = var.nomad_addr
+    NOMAD_CACERT      = var.ca_file
+    NOMAD_CLIENT_CERT = var.cert_file
+    NOMAD_CLIENT_KEY  = var.key_file
+    NOMAD_TOKEN       = var.nomad_token
+    SERVERS_VERSION   = var.servers_version
+    CLIENTS_VERSION   = var.clients_version
+  }
+
+  scripts = [
+    abspath("${path.module}/scripts/versions.sh"),
+  ]
+} 
+

enos/modules/test_cluster_health/scripts/servers.sh

@@ -23,8 +23,18 @@ if [ "$servers_length" -ne "$SERVER_COUNT" ]; then
     error_exit "Unexpected number of servers are alive: $servers_length\n$(echo $servers | jq '.[] | select(.Status != "alive") | .Name')"
 fi
 
-if [ $(echo "$running_servers" | jq -r "map(.last_log_index ) | unique | length == 1") != "true" ]; then
-    error_exit "Servers not up to date"
-fi
+# Quality: nomad_agent_info_self: A GET call to /v1/agent/self against every server returns the same last_log_index for all of them"
+
+last_index=""
+
+for ip in $SERVERS; do
+
+    last_log_index=$(nomad agent-info -address "https://$ip:4646" -json | jq -r '.stats.raft.last_log_index')
+    if [ -n "$last_index" ] && [ "$last_log_index" -ne "$last_index" ]; then
+        error_exit "Servers not on the same index. $ip on  index: $last_index,  previous read index: $last_log_index"
+    fi
+
+    last_index="$last_log_index"
+done
 
 echo "All SERVERS are alive and up to date."

enos/modules/test_cluster_health/scripts/versions.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -euo pipefail
+
+error_exit() {
+    printf 'Error: %s' "${1}" 
+    exit 1
+}
+
+# Servers version
+server_versions=$(nomad server members -json | jq -r '[.[] | select(.Status == "alive") | .Tags.build] | unique')
+
+if [ "$(echo "$server_versions" | jq 'length')" -eq 0 ]; then
+    error_exit "Unable to get servers version"
+fi
+
+if [ "$(echo "$server_versions" | jq 'length')" -ne 1 ]; then
+    error_exit "Servers are running different versions: $(echo "$server_versions" | jq -c '.')"
+fi
+
+final_version=$(echo "$server_versions" | jq -r '.[0]'| xargs)
+SERVERS_VERSION=$(echo "$SERVERS_VERSION" | xargs)
+
+if [ "$final_version" != "$SERVERS_VERSION" ]; then
+    error_exit "Servers are not running the correct version. Found: $final_version, Expected: $SERVERS_VERSION"
+fi
+
+echo "All servers are running Nomad version $SERVERS_VERSION"
+
+# Clients version
+clients_versions=$(nomad node status -json | jq -r '[.[] | select(.Status == "ready") | .Version] | unique')
+
+
+if [ "$(echo "$clients_versions" | jq 'length')" -eq 0 ]; then
+    error_exit "Unable to get clients version"
+fi
+
+
+if [ "$(echo "$clients_versions" | jq 'length')" -ne 1 ]; then
+    error_exit "Clients are running different versions: $(echo "$clients_versions" | jq -c '.')"
+fi
+
+final_version=$(echo "$clients_versions" | jq -r '.[0]'| xargs)
+CLIENTS_VERSION=$(echo "$CLIENTS_VERSION" | xargs)
+
+if [ "$final_version" != "$CLIENTS_VERSION" ]; then
+    error_exit "Clients are not running the correct version. Found: $final_version, Expected: $CLIENTS_VERSION"
+fi
+
+echo "All clients are running Nomad version $CLIENTS_VERSION"

enos/modules/test_cluster_health/variables.tf

@@ -45,3 +45,18 @@ variable "jobs_count" {
 variable "alloc_count" {
   description = "Number of allocation that should be running in the cluster"
 }
+
+variable "clients_version" {
+  description = "Binary version running on the clients"
+  type        = string
+}
+
+variable "servers_version" {
+  description = "Binary version running on the servers"
+  type        = string
+}
+
+variable "servers" {
+  description = "List of public IP address of the nomad servers"
+  type        = list
+}