Exit if the hostengine connection goes down (NVIDIA#110)

Rather than try to reconnect, simply fail and let controller (k8s, etc) determine whether to restart or not.
nicklausbrown · Oct 21, 2022 · 7ca01a8 · 7ca01a8
1 parent 9d6f0cd
commit 7ca01a8
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 1 deletion.
diff --git a/go.mod b/go.mod
@@ -28,7 +28,7 @@ replace (
 )
 
 require (
-	github.com/NVIDIA/go-dcgm v0.0.0-20220929160432-a45eb235ae9a
+	github.com/NVIDIA/go-dcgm v0.0.0-20221020122955-3241b88a3175
 	github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20211102125545-5a2c58442e48 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/sirupsen/logrus v1.7.0

diff --git a/go.sum b/go.sum
@@ -72,6 +72,8 @@ github.com/NVIDIA/go-dcgm v0.0.0-20220728130116-32999852818d h1:3TQlcJb+uuhuMrBi
 github.com/NVIDIA/go-dcgm v0.0.0-20220728130116-32999852818d/go.mod h1:atKWXstYFllLarJucBxnt9ivFIPe26Y6S4JQvzqeSr8=
 github.com/NVIDIA/go-dcgm v0.0.0-20220929160432-a45eb235ae9a h1:M5MXgKCaNuYjvZVnstPfTd05aUG1TZvOX01mtNYJJ/4=
 github.com/NVIDIA/go-dcgm v0.0.0-20220929160432-a45eb235ae9a/go.mod h1:atKWXstYFllLarJucBxnt9ivFIPe26Y6S4JQvzqeSr8=
+github.com/NVIDIA/go-dcgm v0.0.0-20221020122955-3241b88a3175 h1:wQcXli1xIvxM/yBmGQPuuaCTPbrr3hkV+1BPevTY86c=
+github.com/NVIDIA/go-dcgm v0.0.0-20221020122955-3241b88a3175/go.mod h1:atKWXstYFllLarJucBxnt9ivFIPe26Y6S4JQvzqeSr8=
 github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20211102125545-5a2c58442e48 h1:JO/JF5CBte9mvATbhoh32swu9erf07ZdLgwFj8u21UQ=
 github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20211102125545-5a2c58442e48/go.mod h1:oKPJa5eOTkWvlT4/Y4D8Nds44Fzmww5HUK+xwO+DwTA=
 github.com/NVIDIA/gpu-monitoring-tools/bindings/go/dcgm v0.0.0-20210325210537-29b4f1784f18/go.mod h1:8qXwltEzU3idjUcVpMOv3FNgxxbDeXZPGMLyc/khWiY=

diff --git a/pkg/dcgmexporter/gpu_collector.go b/pkg/dcgmexporter/gpu_collector.go
@@ -19,6 +19,7 @@ package dcgmexporter
 import (
 	"fmt"
 	"github.com/NVIDIA/go-dcgm/pkg/dcgm"
+	"github.com/sirupsen/logrus"
 	"os"
 )
 
@@ -69,6 +70,11 @@ func (c *DCGMCollector) GetMetrics() ([][]Metric, error) {
 	for i, mi := range monitoringInfo {
 		vals, err := dcgm.EntityGetLatestValues(mi.Entity.EntityGroupId, mi.Entity.EntityId, c.DeviceFields)
 		if err != nil {
+			if derr, ok := err.(*dcgm.DcgmError); ok {
+				if derr.Code == dcgm.DCGM_ST_CONNECTION_NOT_VALID {
+					logrus.Fatal("Could not retrieve metrics: ", err)
+				}
+			}
 			return nil, err
 		}
 

diff --git a/pkg/dcgmexporter/pipeline.go b/pkg/dcgmexporter/pipeline.go
@@ -93,6 +93,8 @@ func (m *MetricsPipeline) Run(out chan string, stop chan interface{}, wg *sync.W
 			o, err := m.run()
 			if err != nil {
 				logrus.Errorf("Failed to collect metrics with error: %v", err)
+				/* flush output rather than output stale data */
+				out <- ""
 				continue
 			}