[testing] Replace script-based tool installation with nix

Previously, binary tools like promtail and prometheus (enabling log and metrics collection) and kube and kind (enabling local kube clusters) were installed using bash scripts. A script-based approach, while simple, requires tedious and error-prone copying and committing to other repos (e.g. subnet-evm and hypersdk) that need the same functionality. Possible to rewrite the scripts to golang which allows usage via `go run` or `go install`. Simple enough if the versions never change, but still requires retrieving binaries, verifying their hashes for security, and potentially dealing with tar archives. Still more complicated if it becomes necessary to ensure the desired version is the one in the path. Switching to [nix](https://nixos.org/), on the other hand, is an established way of solving the problem that doesn't involve maintaining code. Installing nix is a one-time operation not too different from homebrew, except that the result has more guarantees of reproducibility across linux and macos.
ava-labs · Feb 1, 2025 · 3d0e9bf · 3d0e9bf
1 parent 44ac37e
commit 3d0e9bf
Show file tree

Hide file tree

Showing 9 changed files with 148 additions and 96 deletions.
diff --git a/.envrc b/.envrc
@@ -1,3 +1,5 @@
+use flake
+
 # Repo-local commands like ginkgo and tmpnetctl
 PATH_add bin
 

diff --git a/.github/actions/run-monitored-tmpnet-cmd/action.yml b/.github/actions/run-monitored-tmpnet-cmd/action.yml
@@ -40,14 +40,16 @@ runs:
       # Only run for the original repo; a forked repo won't have access to the monitoring credentials
       if: (inputs.prometheus_username != '')
       shell: bash
-      run: bash -x ./scripts/run_prometheus.sh
+      # Assumes calling project has a nix flake that ensures a compatible prometheus
+      run: nix develop --impure --command bash -x ./scripts/run_prometheus.sh
       env:
         PROMETHEUS_USERNAME: ${{ inputs.prometheus_username }}
         PROMETHEUS_PASSWORD: ${{ inputs.prometheus_password }}
     - name: Start promtail
       if: (inputs.prometheus_username != '')
       shell: bash
-      run: bash -x ./scripts/run_promtail.sh
+      # Assumes calling project has a nix flake that ensures a compatible promtail
+      run: nix develop --impure --command bash -x ./scripts/run_promtail.sh
       env:
         LOKI_USERNAME: ${{ inputs.loki_username }}
         LOKI_PASSWORD: ${{ inputs.loki_password }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -51,6 +51,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
+      - uses: cachix/install-nix-action@v30
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - run: nix build
       - name: Build AvalancheGo Binary
         shell: bash
         run: ./scripts/build.sh -r
@@ -72,6 +76,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
+      - uses: cachix/install-nix-action@v30
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - run: nix build
       - name: Build AvalancheGo Binary
         shell: bash
         run: ./scripts/build.sh -r
@@ -93,6 +101,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
+      - uses: cachix/install-nix-action@v30
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - run: nix build
       - name: Build AvalancheGo Binary
         shell: bash
         run: ./scripts/build.sh
@@ -230,6 +242,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-go-for-project
+      - uses: cachix/install-nix-action@v30
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+      - run: nix build
       - name: Run e2e tests
         shell: bash
         run: bash -x ./scripts/tests.e2e.bootstrap_monitor.sh
diff --git a/.gitignore b/.gitignore
@@ -62,3 +62,5 @@ tests/upgrade/upgrade.test
 vendor
 
 **/testdata
+
+.direnv
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -0,0 +1,75 @@
+{
+  # To use:
+  #  - install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix
+  #  - run `nix develop` or use direnv (https://direnv.net/)
+  #    - for quieter direnv output, set `export DIRENV_LOG_FORMAT=`
+
+  description = "AvalancheGo development environment";
+
+  # Flake inputs
+  inputs = {
+    nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2405.*.tar.gz";
+  };
+
+  # Flake outputs
+  outputs = { self, nixpkgs }:
+    let
+      # Systems supported
+      allSystems = [
+        "x86_64-linux" # 64-bit Intel/AMD Linux
+        "aarch64-linux" # 64-bit ARM Linux
+        "x86_64-darwin" # 64-bit Intel macOS
+        "aarch64-darwin" # 64-bit ARM macOS
+      ];
+
+      # Helper to provide system-specific attributes
+      forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f {
+        pkgs = import nixpkgs { inherit system; };
+      });
+    in
+    {
+      # Development environment output
+      devShells = forAllSystems ({ pkgs }: {
+        default = pkgs.mkShell {
+          # The Nix packages provided in the environment
+          packages = with pkgs; [
+            # Monitoring tools
+            promtail                                   # Loki log shipper
+            prometheus                                 # Metrics collector
+
+            # Kube tools
+            kubectl                                    # Kubernetes CLI
+            kind                                       # Kubernetes-in-Docker
+            kubernetes-helm                            # Helm CLI (Kubernetes package manager)
+            self.packages.${system}.kind-with-registry # Script installing kind configured with a local registry
+          ];
+        };
+      });
+
+      # Package to install the kind-with-registry script
+      packages = forAllSystems ({ pkgs }: {
+        kind-with-registry = pkgs.stdenv.mkDerivation {
+          pname = "kind-with-registry";
+          version = "1.0.0";
+
+          src = pkgs.fetchurl {
+            url = "https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh";
+            sha256 = "0gri0x0ygcwmz8l4h6zzsvydw8rsh7qa8p5218d4hncm363i81hv";
+          };
+
+          phases = [ "installPhase" ];
+
+          installPhase = ''
+            mkdir -p $out/bin
+            install -m755 $src $out/bin/kind-with-registry.sh
+          '';
+
+          meta = with pkgs.lib; {
+            description = "Script to set up kind with a local registry";
+            license = licenses.mit;
+            maintainers = with maintainers; [ "maru-ava" ];
+          };
+        };
+      });
+    };
+}
diff --git a/scripts/run_prometheus.sh b/scripts/run_prometheus.sh
@@ -20,6 +20,14 @@ if ! [[ "$0" =~ scripts/run_prometheus.sh ]]; then
   exit 255
 fi
 
+CMD=prometheus
+
+if ! command -v "${CMD}" &> /dev/null; then
+  echo "prometheus not found, have you run `nix develop`?"
+  echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix"
+  exit 1
+fi
+
 PROMETHEUS_WORKING_DIR="${HOME}/.tmpnet/prometheus"
 PIDFILE="${PROMETHEUS_WORKING_DIR}"/run.pid
 
@@ -48,39 +56,6 @@ if [[ -z "${PROMETHEUS_PASSWORD}" ]]; then
   exit 1
 fi
 
-# This was the LTS version when this script was written. Probably not
-# much reason to update it unless something breaks since the usage
-# here is only to collect metrics from temporary networks.
-VERSION="2.45.3"
-
-# Ensure the prometheus command is locally available
-CMD=prometheus
-if ! command -v "${CMD}" &> /dev/null; then
-  # Try to use a local version
-  CMD="${PWD}/bin/prometheus"
-  if ! command -v "${CMD}" &> /dev/null; then
-    echo "prometheus not found, attempting to install..."
-
-    GOOS="$(go env GOOS)"
-    GOARCH="$(go env GOARCH)"
-    if [[ "${GOOS}" == "darwin" && "${GOARCH}" == "arm64" ]]; then
-      echo "On macos, only amd64 binaries are available so rosetta is required on apple silicon machines."
-      echo "To avoid using rosetta, install via homebrew: brew install prometheus"
-    fi
-    if [[ "${GOOS}" == "linux" && "${GOARCH}" != "amd64" ]]; then
-        echo "On linux, only amd64 binaries are available. Manual installation of prometheus is required."
-        exit 1
-    fi
-
-    # Install the specified release
-    PROMETHEUS_FILE="prometheus-${VERSION}.${GOOS}-amd64"
-    URL="https://github.com/prometheus/prometheus/releases/download/v${VERSION}/${PROMETHEUS_FILE}.tar.gz"
-    curl -s -L "${URL}" | tar zxv -C /tmp > /dev/null
-    mkdir -p "$(dirname "${CMD}")"
-    cp /tmp/"${PROMETHEUS_FILE}/prometheus" "${CMD}"
-  fi
-fi
-
 # Configure prometheus
 FILE_SD_PATH="${PROMETHEUS_WORKING_DIR}/file_sd_configs"
 mkdir -p "${FILE_SD_PATH}"

diff --git a/scripts/run_promtail.sh b/scripts/run_promtail.sh
@@ -19,6 +19,14 @@ if ! [[ "$0" =~ scripts/run_promtail.sh ]]; then
   exit 255
 fi
 
+CMD=promtail
+
+if ! command -v "${CMD}" &> /dev/null; then
+  echo "promtail not found, have you run `nix develop`?"
+  echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix"
+  exit 1
+fi
+
 PROMTAIL_WORKING_DIR="${HOME}/.tmpnet/promtail"
 PIDFILE="${PROMTAIL_WORKING_DIR}"/run.pid
 
@@ -47,33 +55,6 @@ if [[ -z "${LOKI_PASSWORD}" ]]; then
   exit 1
 fi
 
-# Version as of this writing
-VERSION="v2.9.5"
-
-# Ensure the promtail command is locally available
-CMD=promtail
-if ! command -v "${CMD}" &> /dev/null; then
-  # Try to use a local version
-  CMD="${PWD}/bin/promtail"
-  if ! command -v "${CMD}" &> /dev/null; then
-    echo "promtail not found, attempting to install..."
-
-    # Determine the platform
-    GOOS="$(go env GOOS)"
-    GOARCH="$(go env GOARCH)"
-    DIST="${GOOS}-${GOARCH}"
-
-    # Install the specified release
-    PROMTAIL_FILE="promtail-${DIST}"
-    ZIP_PATH="/tmp/${PROMTAIL_FILE}.zip"
-    BIN_DIR="$(dirname "${CMD}")"
-    URL="https://github.com/grafana/loki/releases/download/${VERSION}/promtail-${DIST}.zip"
-    curl -L -o "${ZIP_PATH}" "${URL}"
-    unzip "${ZIP_PATH}" -d "${BIN_DIR}"
-    mv "${BIN_DIR}/${PROMTAIL_FILE}" "${CMD}"
-  fi
-fi
-
 # Configure promtail
 FILE_SD_PATH="${PROMTAIL_WORKING_DIR}/file_sd_configs"
 mkdir -p "${FILE_SD_PATH}"

diff --git a/scripts/tests.e2e.bootstrap_monitor.sh b/scripts/tests.e2e.bootstrap_monitor.sh
@@ -9,40 +9,14 @@ if ! [[ "$0" =~ scripts/tests.e2e.bootstrap_monitor.sh ]]; then
   exit 255
 fi
 
-GOOS="$(go env GOOS)"
-GOARCH="$(go env GOARCH)"
+CMD=kind-with-registry.sh
 
-function ensure_command {
-  local cmd=$1
-  local install_uri=$2
-
-  if ! command -v "${cmd}" &> /dev/null; then
-    # Try to use a local version
-    local local_cmd="${PWD}/bin/${cmd}"
-    mkdir -p "${PWD}/bin"
-    if ! command -v "${local_cmd}" &> /dev/null; then
-      echo "${cmd} not found, attempting to install..."
-      curl -L -o "${local_cmd}" "${install_uri}"
-      # TODO(marun) Optionally validate the binary against published checksum
-      chmod +x "${local_cmd}"
-    fi
-  fi
-}
-
-# Ensure the kubectl command is available
-KUBECTL_VERSION=v1.30.2
-ensure_command kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${GOOS}/${GOARCH}/kubectl"
-
-# Ensure the kind command is available
-KIND_VERSION=v0.23.0
-ensure_command kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-${GOOS}-${GOARCH}"
-
-# Ensure the kind-with-registry command is available
-ensure_command "kind-with-registry.sh" "https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh"
+if ! command -v "${CMD}" &> /dev/null; then
+  echo "kind-with-registry.sh not found, have you run `nix develop`?"
+  echo "To install nix: https://github.com/DeterminateSystems/nix-installer?tab=readme-ov-file#install-nix"
+  exit 1
+fi
 
-# Deploy a kind cluster with a local registry. Include the local bin in the path to
-# ensure locally installed kind and kubectl are available since the script expects to
-# call them without a qualifying path.
-PATH="${PWD}/bin:$PATH" bash -x "${PWD}/bin/kind-with-registry.sh"
+"${CMD}"
 
-KUBECONFIG="$HOME/.kube/config" PATH="${PWD}/bin:$PATH" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e
+KUBECONFIG="$HOME/.kube/config" ./bin/ginkgo -v ./tests/fixture/bootstrapmonitor/e2e
Original file line number	Diff line number	Diff line change
Expand Up		@@ -62,3 +62,5 @@ tests/upgrade/upgrade.test
		vendor

		**/testdata

		.direnv