From 9d077eb491285c74ec2a695a95738e2023c0e88b Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Tue, 3 Sep 2024 17:07:28 -0400
Subject: [PATCH 01/20] Add working spack build with build cache.

---
 .github/workflows/spack_build.yml         | 168 ++++++++++++++++++++++
 .github/workflows/spack_build.yml_archive |  47 ------
 2 files changed, 168 insertions(+), 47 deletions(-)
 create mode 100644 .github/workflows/spack_build.yml
 delete mode 100644 .github/workflows/spack_build.yml_archive

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
new file mode 100644
index 00000000..f388647f
--- /dev/null
+++ b/.github/workflows/spack_build.yml
@@ -0,0 +1,168 @@
+# https://spack.readthedocs.io/en/latest/binary_caches.html#spack-build-cache-for-github-actions
+name: Spack Builds (Ubuntu x86_64 Buildcache)
+
+on: [pull_request]
+
+env:
+  SPACK_COLOR: always
+  REGISTRY: ghcr.io/llnl
+  # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
+  IMAGE_NAME: hiop
+  USERNAME: hiop-bot
+  BASE_VERSION: ubuntu-24.04-fortran
+
+jobs:
+  base_image_build:
+    runs-on: ubuntu-24.04
+    permissions:
+      packages: write
+      contents: read
+
+    name: Build Custom Base Image
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Once we move submodule deps into spack, we can do some more builds
+          # Also need to change build script to use spack from base image
+          submodules: true
+
+      # No GHCR base image with skopeo, so this will do...
+      - name: "Set up skopeo"
+        uses: warjiang/setup-skopeo@v0.1.3
+        with:
+          version: latest
+
+      # Use skopeo to check for image for convenience
+      - name: Check for existing base images
+        run: |
+          set -e
+          CONTAINER_TAG=${{ env.BASE_VERSION }}
+          OCI_URL="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }}"
+          echo Checking for CONTAINER_TAG $CONTAINER_TAG
+          skopeo inspect \
+              docker://$OCI_URL \
+              --raw \
+              --creds "${{ env.USERNAME }}:${{ secrets.GITHUB_TOKEN }}" \
+            > /dev/null && echo "Image already exists. Please bump version." && exit 0
+          echo "IMAGE_EXISTS=false" >> $GITHUB_ENV
+
+      # Need to build custom base image with gfortran
+      - name: Create Dockerfile heredoc
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        run: |
+          cat << EOF > Dockerfile
+          FROM ubuntu:24.04
+          RUN apt-get update && \
+              apt-get install -y --no-install-recommends \
+                software-properties-common \
+                gpg-agent \
+                openssh-client \
+                openssh-server \
+              && rm -rf /var/lib/apt/lists/*
+          RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
+              apt-get install -y --no-install-recommends \
+                gfortran \
+                gcc \
+                libstdc++6 \
+              && rm -rf /var/lib/apt/lists/*
+          EOF
+
+      # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: org.opencontainers.image.version=${{ env.BASE_VERSION }}
+
+      - name: Build and push Docker base image
+        if: ${{ env.IMAGE_EXISTS == 'false' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  hiop_spack_builds:
+    needs: base_image_build
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+    container: spack/ubuntu-jammy:latest
+    strategy:
+      matrix:
+        spack_spec:
+          - hiop@develop+mpi~raja~shared~kron~sparse ^openmpi ^libevent~openssl
+          - hiop@develop~mpi~raja~shared~kron~sparse ^libevent~openssl
+          - hiop@develop~mpi+raja~shared~kron~sparse ^libevent~openssl
+
+            # We will need coinhsl for this, but what are the rules for using
+            # a coinhsl tarball?
+            # - hiop@develop~mpi~raja~shared~kron+sparse
+
+    name: Build HiOp with Spack
+
+    steps:
+      - name: Setup Spack
+        run: echo "/opt/spack/bin" >> "$GITHUB_PATH"
+
+      - name: Create heredoc spack.yaml
+        run: |
+          spack debug report
+          cat << EOF > spack.yaml
+          spack:
+            specs:
+            - ${{ matrix.spack_spec }} target=x86_64_v2
+            concretizer:
+              reuse: dependencies
+            config:
+              source_cache: $SPACK_CACHE/source_cache
+              misc_cache: $SPACK_CACHE/misc_cache
+              build_stage: $SPACK_CACHE/build_stage
+              install_tree:
+                root: /opt/spack
+                padded_length: False
+            mirrors:
+              local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            # spack: https://binaries.spack.io/develop
+            packages:
+              all:
+                require: "%gcc"
+          EOF
+
+      - name: Configure GHCR mirror
+        run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
+
+      - name: Trust keys
+        run: spacke -e . buildcache keys --install --trust
+
+      - name: Find external packages
+        run: spack -e . external find --all --exclude python
+
+      - name: Spack develop HiOp
+        run: spack -e . develop --path=$(pwd) hiop@develop
+
+      - name: Concretize
+        run: spack -e . concretize --fresh
+
+      - name: Install
+        run: spack -e . install --no-check-signature --fail-fast --keep-stage
+
+      - name: Test Build
+        run: cd $(spack -e . location --build-dir hiop@develop) && ctest -VV
+
+      - name: Push binaries to buildcache
+        run: |
+          spack -e . buildcache push --force --base-image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }} --unsigned --update-index local-buildcache
+        if: ${{ !cancelled() }}
diff --git a/.github/workflows/spack_build.yml_archive b/.github/workflows/spack_build.yml_archive
deleted file mode 100644
index fa75c4c3..00000000
--- a/.github/workflows/spack_build.yml_archive
+++ /dev/null
@@ -1,47 +0,0 @@
-name: Spack Builds
-
-on: [push]
-
-jobs:
-  hiop_spack_builds:
-    runs-on: ubuntu-22.04
-    container: spack/ubuntu-jammy:latest
-    strategy:
-      matrix:
-        spack_spec:
-          - hiop@develop+mpi~raja~shared~kron~sparse ^openmpi ^libevent~openssl
-          - hiop@develop~mpi~raja~shared~kron~sparse ^libevent~openssl
-          - hiop@develop~mpi+raja~shared~kron~sparse ^libevent~openssl
-
-            # We will need coinhsl for this, but what are the rules for using
-            # a coinhsl tarball?
-            # - hiop@develop~mpi~raja~shared~kron+sparse
-
-    name: Build HiOp with Spack
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-
-      - name: Build Environment
-        env:
-          SPACK_SPEC: ${{ matrix.spack_spec }}
-        run: |
-          ls && pwd
-          . /opt/spack/share/spack/setup-env.sh
-          spack debug report
-          # Just use the public mirror to bootstrap concretizer
-          # https://spack.io/spack-binary-packages
-          spack mirror add spack_public_mirror https://binaries.spack.io/develop
-          spack buildcache keys --install --trust
-          # Need to create an environment to install hiop in the action's branch
-          spack env create -d ./spack-env
-          spack env activate ./spack-env
-          spack add $SPACK_SPEC target=x86_64
-          spack develop --path $(pwd) --no-clone hiop@develop
-          spack external find --all --exclude python
-          spack concretize --reuse
-          git config --global --add safe.directory $(pwd)
-          spack --stacktrace install --fail-fast
-

From 3bcd263216695105a7bab3cb41c075cd5d0bb315 Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Tue, 3 Sep 2024 17:10:24 -0400
Subject: [PATCH 02/20] Fix spacke => spack typo.

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index f388647f..7b599850 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -145,7 +145,7 @@ jobs:
         run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
 
       - name: Trust keys
-        run: spacke -e . buildcache keys --install --trust
+        run: spack -e . buildcache keys --install --trust
 
       - name: Find external packages
         run: spack -e . external find --all --exclude python

From 6f8c20128d5d2bf72b5542309fb9de8819952e5c Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 6 Sep 2024 11:26:58 -0400
Subject: [PATCH 03/20] Also exclude curl from build.

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 7b599850..179d7827 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -148,7 +148,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Find external packages
-        run: spack -e . external find --all --exclude python
+        run: spack -e . external find --all --exclude python --exclude curl
 
       - name: Spack develop HiOp
         run: spack -e . develop --path=$(pwd) hiop@develop

From cbbe7d6d61789d8b6e31987bf2e7f556f6e7b333 Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 6 Sep 2024 11:37:10 -0400
Subject: [PATCH 04/20] Also exclude openssl.

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 179d7827..19a2c0f4 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -148,7 +148,7 @@ jobs:
         run: spack -e . buildcache keys --install --trust
 
       - name: Find external packages
-        run: spack -e . external find --all --exclude python --exclude curl
+        run: spack -e . external find --all --exclude python --exclude curl --exclude openssl
 
       - name: Spack develop HiOp
         run: spack -e . develop --path=$(pwd) hiop@develop

From 50a045f775f2215672989e37b5c1c697a67d20f2 Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 13 Sep 2024 13:48:17 -0400
Subject: [PATCH 05/20] Use custom spack for with fix.

---
 .github/workflows/spack_build.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 19a2c0f4..4840ab3e 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -99,7 +99,6 @@ jobs:
     permissions:
       packages: write
       contents: read
-    container: spack/ubuntu-jammy:latest
     strategy:
       matrix:
         spack_spec:
@@ -115,7 +114,13 @@ jobs:
 
     steps:
       - name: Setup Spack
-        run: echo "/opt/spack/bin" >> "$GITHUB_PATH"
+        run: |
+          # TODO: Move back to spack@develop once patch in in develop
+          git clone https://github.com/cameronrutherford/spack.git
+          cd spack
+          git checkout 9bbf4405306ba85ed66907c4e1cd3a2ce980dd38
+          cd -
+          echo $(pwd)/spack/opt/spack/bin >> "$GITHUB_PATH"
 
       - name: Create heredoc spack.yaml
         run: |

From 45a452a38e3daee156f4399cad529a7ece5faefc Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 13 Sep 2024 13:51:33 -0400
Subject: [PATCH 06/20] Change how spack is configured.

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 4840ab3e..ef76d94d 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -120,7 +120,7 @@ jobs:
           cd spack
           git checkout 9bbf4405306ba85ed66907c4e1cd3a2ce980dd38
           cd -
-          echo $(pwd)/spack/opt/spack/bin >> "$GITHUB_PATH"
+          source ./spack/share/spack/setup-env.sh
 
       - name: Create heredoc spack.yaml
         run: |

From 5d5f479975fe055727ed0ef87a74f561a50183ca Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 13 Sep 2024 13:56:10 -0400
Subject: [PATCH 07/20] Fix configuring spack command.

---
 .github/workflows/spack_build.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index ef76d94d..c1758a36 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -113,14 +113,16 @@ jobs:
     name: Build HiOp with Spack
 
     steps:
-      - name: Setup Spack
+      - name: Clone Spack
         run: |
           # TODO: Move back to spack@develop once patch in in develop
           git clone https://github.com/cameronrutherford/spack.git
-          cd spack
+          pushd spack
           git checkout 9bbf4405306ba85ed66907c4e1cd3a2ce980dd38
-          cd -
-          source ./spack/share/spack/setup-env.sh
+          popd
+
+      - name: Setup Spack
+        run: echo "$PWD/spack/bin" >> "$GITHUB_PATH"
 
       - name: Create heredoc spack.yaml
         run: |

From fb41bde146804986125646afe95cbfd8e298a51b Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 13 Sep 2024 13:59:00 -0400
Subject: [PATCH 08/20] Fix cache path and TMPDIR.

---
 .github/workflows/spack_build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index c1758a36..e54cac5f 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -6,6 +6,10 @@ on: [pull_request]
 env:
   SPACK_COLOR: always
   REGISTRY: ghcr.io/llnl
+  SPACK_CACHE: /opt/spack-cache
+  tempdir: /opt/spack-cache
+  TMP: /opt/spack-cache
+  TMPDIR: /opt/spack-cache
   # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
   IMAGE_NAME: hiop
   USERNAME: hiop-bot

From a257819f408fa24bc1bdb660862d7215a86a77ac Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Fri, 13 Sep 2024 15:16:09 -0400
Subject: [PATCH 09/20] Add output of full log on error and add verbosity to
 HiOp build.

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index e54cac5f..a8aa8e6e 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -168,7 +168,7 @@ jobs:
         run: spack -e . concretize --fresh
 
       - name: Install
-        run: spack -e . install --no-check-signature --fail-fast --keep-stage
+        run: spack -e . install --no-check-signature --fail-fast --keep-stage --show-log-on-error --verbose
 
       - name: Test Build
         run: cd $(spack -e . location --build-dir hiop@develop) && ctest -VV

From 2792180bce2ce83d1f8d3b7333310db5e7dfa307 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Wed, 18 Sep 2024 10:41:01 -0400
Subject: [PATCH 10/20] Force openblas on all builds

---
 .github/workflows/spack_build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index a8aa8e6e..1939aa26 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -106,9 +106,9 @@ jobs:
     strategy:
       matrix:
         spack_spec:
-          - hiop@develop+mpi~raja~shared~kron~sparse ^openmpi ^libevent~openssl
-          - hiop@develop~mpi~raja~shared~kron~sparse ^libevent~openssl
-          - hiop@develop~mpi+raja~shared~kron~sparse ^libevent~openssl
+          - hiop@develop+mpi~raja~shared~kron~sparse ^openblas ^openmpi ^libevent~openssl
+          - hiop@develop~mpi~raja~shared~kron~sparse ^openblas ^libevent~openssl
+          - hiop@develop~mpi+raja~shared~kron~sparse ^openblas ^libevent~openssl
 
             # We will need coinhsl for this, but what are the rules for using
             # a coinhsl tarball?

From a6a05690910cf351a7157ec6850a8d04af17ef02 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Wed, 18 Sep 2024 16:38:55 -0400
Subject: [PATCH 11/20] Update spack_build.yml

---
 .github/workflows/spack_build.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 1939aa26..49face4e 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -167,8 +167,13 @@ jobs:
       - name: Concretize
         run: spack -e . concretize --fresh
 
-      - name: Install
-        run: spack -e . install --no-check-signature --fail-fast --keep-stage --show-log-on-error --verbose
+      - name: Install Dependencies
+        run: spack -e . install --no-check-signature --fail-fast --show-log-on-error --verbose --only dependencies
+
+      - name: Install HiOp
+        run: |
+          ls -al
+          spack -d -e . install  --keep-stage --verbose --show-log-on-error --only package
 
       - name: Test Build
         run: cd $(spack -e . location --build-dir hiop@develop) && ctest -VV

From 4306636bac31e8318b651d12048c9585f7392163 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 10:32:19 -0400
Subject: [PATCH 12/20] Update spack_build.yml with checkout in right place

---
 .github/workflows/spack_build.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 49face4e..bd406bc4 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -24,13 +24,6 @@ jobs:
 
     name: Build Custom Base Image
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          # Once we move submodule deps into spack, we can do some more builds
-          # Also need to change build script to use spack from base image
-          submodules: true
-
       # No GHCR base image with skopeo, so this will do...
       - name: "Set up skopeo"
         uses: warjiang/setup-skopeo@v0.1.3
@@ -117,6 +110,13 @@ jobs:
     name: Build HiOp with Spack
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Once we move submodule deps into spack, we can do some more builds
+          # Also need to change build script to use spack from base image
+          submodules: true
+        
       - name: Clone Spack
         run: |
           # TODO: Move back to spack@develop once patch in in develop

From d5f805f8f9a36d210963f131086daf1f28d7f0f0 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 10:47:33 -0400
Subject: [PATCH 13/20] Load openmpi, and use git.version spack version

---
 .github/workflows/spack_build.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index bd406bc4..a0e96831 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -162,7 +162,7 @@ jobs:
         run: spack -e . external find --all --exclude python --exclude curl --exclude openssl
 
       - name: Spack develop HiOp
-        run: spack -e . develop --path=$(pwd) hiop@develop
+        run: spack -e . develop --path=$(pwd) hiop@develop@git."${{ github.head_ref || github.ref_name }}"=develop
 
       - name: Concretize
         run: spack -e . concretize --fresh
@@ -176,7 +176,11 @@ jobs:
           spack -d -e . install  --keep-stage --verbose --show-log-on-error --only package
 
       - name: Test Build
-        run: cd $(spack -e . location --build-dir hiop@develop) && ctest -VV
+        run: |
+          # Not all pipelines have `+mpi`, so this command might fail
+          spack -e . load openmpi || true
+          cd $(spack -e . location --build-dir hiop@develop)
+          ctest -VV
 
       - name: Push binaries to buildcache
         run: |

From c8a965181526323cd626e41f0d19dfe20b4bb4a5 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 10:49:22 -0400
Subject: [PATCH 14/20] Update spack_build.yml with correct Spack commit SHA

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index a0e96831..0d3a78d7 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -122,7 +122,7 @@ jobs:
           # TODO: Move back to spack@develop once patch in in develop
           git clone https://github.com/cameronrutherford/spack.git
           pushd spack
-          git checkout 9bbf4405306ba85ed66907c4e1cd3a2ce980dd38
+          git checkout 838f2b6708a28cae7b1c1de1f1e1f29a1bf2ee0d
           popd
 
       - name: Setup Spack

From 64d4a1d693e48cd936358132f34fe15a9eb0b2b1 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 10:55:24 -0400
Subject: [PATCH 15/20] Update spack_build.yml with correct spack vesion syntax

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 0d3a78d7..af566bba 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -162,7 +162,7 @@ jobs:
         run: spack -e . external find --all --exclude python --exclude curl --exclude openssl
 
       - name: Spack develop HiOp
-        run: spack -e . develop --path=$(pwd) hiop@develop@git."${{ github.head_ref || github.ref_name }}"=develop
+        run: spack -e . develop --path=$(pwd) hiop@git."${{ github.head_ref || github.ref_name }}"=develop
 
       - name: Concretize
         run: spack -e . concretize --fresh

From 692742a87cb8ff2a1b7236c092f03b3badcea180 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 11:06:32 -0400
Subject: [PATCH 16/20] Update spack_build.yml with `--sh` for loading openmpi

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index af566bba..ca374e3e 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -178,7 +178,7 @@ jobs:
       - name: Test Build
         run: |
           # Not all pipelines have `+mpi`, so this command might fail
-          spack -e . load openmpi || true
+          spack -e . load --sh openmpi || true
           cd $(spack -e . location --build-dir hiop@develop)
           ctest -VV
 

From 2cfa5fcad305d163b558d4556b77a532315bec0d Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 11:20:36 -0400
Subject: [PATCH 17/20] Update spack_build.yml

---
 .github/workflows/spack_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index ca374e3e..79cfa124 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -173,7 +173,7 @@ jobs:
       - name: Install HiOp
         run: |
           ls -al
-          spack -d -e . install  --keep-stage --verbose --show-log-on-error --only package
+          spack -d -e . install  --keep-stage --verbose --show-log-on-error --only package --no-cache
 
       - name: Test Build
         run: |

From f4b8db872289bf6881b5bb6564ca8f43752f4794 Mon Sep 17 00:00:00 2001
From: "Rutherford, Cameron" <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 19:39:43 -0400
Subject: [PATCH 18/20] Load MPI through printed variables from spack.

---
 .github/workflows/spack_build.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index 79cfa124..f10ef4c1 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -116,7 +116,7 @@ jobs:
           # Once we move submodule deps into spack, we can do some more builds
           # Also need to change build script to use spack from base image
           submodules: true
-        
+
       - name: Clone Spack
         run: |
           # TODO: Move back to spack@develop once patch in in develop
@@ -178,7 +178,9 @@ jobs:
       - name: Test Build
         run: |
           # Not all pipelines have `+mpi`, so this command might fail
-          spack -e . load --sh openmpi || true
+          # Command also only prints shell commands, need to source
+          spack -e . load --sh openmpi > env.txt || [ rm env.txt || true ]
+          source env.txt || true
           cd $(spack -e . location --build-dir hiop@develop)
           ctest -VV
 

From bb45d9ea80fc551078dd7ad5c2ec51eab94c81fa Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 19:59:03 -0400
Subject: [PATCH 19/20] Update .github/workflows/spack_build.yml

---
 .github/workflows/spack_build.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index f10ef4c1..e0bd744d 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -119,10 +119,7 @@ jobs:
 
       - name: Clone Spack
         run: |
-          # TODO: Move back to spack@develop once patch in in develop
-          git clone https://github.com/cameronrutherford/spack.git
-          pushd spack
-          git checkout 838f2b6708a28cae7b1c1de1f1e1f29a1bf2ee0d
+          git clone https://github.com/spack/spack.git
           popd
 
       - name: Setup Spack

From 7ea9f5065b6f5470058c8460962fa349191b4cb9 Mon Sep 17 00:00:00 2001
From: Cameron Rutherford <robert.rutherford@pnnl.gov>
Date: Thu, 19 Sep 2024 20:01:24 -0400
Subject: [PATCH 20/20] Update spack_build.yml

---
 .github/workflows/spack_build.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
index e0bd744d..bc7af238 100644
--- a/.github/workflows/spack_build.yml
+++ b/.github/workflows/spack_build.yml
@@ -118,9 +118,7 @@ jobs:
           submodules: true
 
       - name: Clone Spack
-        run: |
-          git clone https://github.com/spack/spack.git
-          popd
+        run: git clone https://github.com/spack/spack.git
 
       - name: Setup Spack
         run: echo "$PWD/spack/bin" >> "$GITHUB_PATH"