From 06c69da8b753f7d0a9ff751a0e4828d83fffb6b0 Mon Sep 17 00:00:00 2001 From: Eugene Fedorenko Date: Wed, 18 Dec 2024 16:57:12 -0800 Subject: [PATCH] http api --- storage/api/grpc/client/client_test.go | 7 +- storage/api/grpc/proto/storage.pb.go | 2 +- storage/api/grpc/server/server.go | 118 +++++++++++++++++-------- storage/go.mod | 31 ++++++- storage/go.sum | 79 +++++++++++++++++ storage/postgres/deployment.go | 23 ----- storage/postgres/postgres_client.go | 6 +- storage/postgres/reconciler.go | 71 +++++++++++++-- 8 files changed, 257 insertions(+), 80 deletions(-) diff --git a/storage/api/grpc/client/client_test.go b/storage/api/grpc/client/client_test.go index c891124..6a94769 100644 --- a/storage/api/grpc/client/client_test.go +++ b/storage/api/grpc/client/client_test.go @@ -167,11 +167,12 @@ func TestUpdateDeployment(t *testing.T) { func TestGetDeploymentState(t *testing.T) { client := NewObservabilityStorageGrpcClient(*serverAddr) deployment_state, err := client.GetDeploymentState(context.Background(), &pb.DeploymentStateRequest{ - ManifestsEndpoint: "https://github.com/KraftHeinz-Org/st-one-data-ingestion-gitops_CI-9232/dev", - CommitId: "f5c23e44b79448c29139632fad1a905c67e4879a", + ManifestsEndpoint: "https://github.com/Org/data-ingestion/dev%", + CommitId: "sha1:a97f94d3f3ede63d08d58ae4cf9f3262d7679c4f", }) // Print the deployment state - t.Logf("DeploymentState: %v", deployment_state) + t.Errorf("DeploymentState: %v", *deployment_state) + if err != nil { t.Errorf("GetDeploymentState() error = %v", err) return diff --git a/storage/api/grpc/proto/storage.pb.go b/storage/api/grpc/proto/storage.pb.go index 8465514..3ff1a0c 100644 --- a/storage/api/grpc/proto/storage.pb.go +++ b/storage/api/grpc/proto/storage.pb.go @@ -957,7 +957,7 @@ type DeploymentState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TotalSubscribers int32 `protobuf:"varint,1,opt,name=total_subscribers,json=totalSubscribers,proto3" json:"total_subscribers,omitempty"` + TotalSubscribers int32 `protobuf:"varint,1,opt,name=total_subscribers,json=totalSubscribers,proto3" json:"total_subscribers"` TotalSucceededSubscribers int32 `protobuf:"varint,2,opt,name=total_succeeded_subscribers,json=totalSucceededSubscribers,proto3" json:"total_succeeded_subscribers,omitempty"` TotalFailedSubscribers int32 `protobuf:"varint,3,opt,name=total_failed_subscribers,json=totalFailedSubscribers,proto3" json:"total_failed_subscribers,omitempty"` TotalInProgressSubscribers int32 `protobuf:"varint,4,opt,name=total_in_progress_subscribers,json=totalInProgressSubscribers,proto3" json:"total_in_progress_subscribers,omitempty"` diff --git a/storage/api/grpc/server/server.go b/storage/api/grpc/server/server.go index 563283e..fd708e6 100644 --- a/storage/api/grpc/server/server.go +++ b/storage/api/grpc/server/server.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" "log" - "net" + "net/http" "os" "strconv" @@ -13,6 +13,9 @@ import ( db "github.com/microsoft/kalypso-observability-hub/storage/postgres" "google.golang.org/grpc" + "github.com/gin-gonic/gin" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" ) @@ -38,6 +41,8 @@ type storageApiServer struct { // make sure that the server implements the interface var _ pb.StorageApiServer = (*storageApiServer)(nil) +var server *storageApiServer + func (s *storageApiServer) UpdateWorkspace(ctx context.Context, workspace *pb.Workspace) (*pb.Workspace, error) { log.Printf("Received Workspace: %v", workspace) @@ -364,12 +369,7 @@ func (s *storageApiServer) GetDeploymentTarget(ctx context.Context, deploymentTa func (s *storageApiServer) GetDeploymentState(ctx context.Context, deploymentStateRequest *pb.DeploymentStateRequest) (*pb.DeploymentState, error) { log.Printf("Received DeploymentStateRequest: %v", deploymentStateRequest) - //Get Reconcilers by ManifestsEndpoint - reconcilers, err := s.dbClient.Query(ctx, db.GetByManifestsEndpoint, deploymentStateRequest.ManifestsEndpoint) - if err != nil { - return nil, err - } - var deployment_state *pb.DeploymentState = &pb.DeploymentState{ + deployment_state := &pb.DeploymentState{ TotalSubscribers: 0, TotalSucceededSubscribers: 0, TotalFailedSubscribers: 0, @@ -379,35 +379,37 @@ func (s *storageApiServer) GetDeploymentState(ctx context.Context, deploymentSta InProgressSubscribers: []*pb.Subscriber{}, } - for _, reconciler := range reconcilers { - deployment_state.TotalSubscribers++ - reconciler_entity := reconciler.(*db.Reconciler) - //Get Deployments by ReconcilerId - deployments_entities, err := s.dbClient.Query(ctx, db.GetByReconcilerId, reconciler_entity.Id) + //Get Reconcilers by ManifestsEndpoint + total_subscribers, err := s.dbClient.Query(ctx, db.CountByManifestsEndpoint, deploymentStateRequest.ManifestsEndpoint) + if err != nil { + return nil, err + } + + deployment_state.TotalSubscribers = total_subscribers.(int32) + + if deployment_state.TotalSubscribers > 0 { + //Get DeploymentStatuses by ManifestsEndpoint and GitopsCommitId + deployment_statuses, err := s.dbClient.Query(ctx, db.CountByStatuses, deploymentStateRequest.ManifestsEndpoint, deploymentStateRequest.CommitId) if err != nil { return nil, err } - //iterate over deployments_entities - for _, deployment := range deployments_entities { - deployment_entity := deployment.(*db.Deployment) - if deployment_entity.GitopsCommitId == deploymentStateRequest.CommitId { + + status := deployment_statuses.(db.StatusStats) + deployment_state.TotalSucceededSubscribers = status.Success + deployment_state.TotalFailedSubscribers = status.Failed + deployment_state.TotalInProgressSubscribers = status.InProgress + + if deployment_state.TotalFailedSubscribers > 0 { + failed_deployments, err := s.dbClient.Query(ctx, db.GetByStatus, deploymentStateRequest.ManifestsEndpoint, deploymentStateRequest.CommitId, "failure") + if err != nil { + return nil, err + } + for _, deployment := range failed_deployments.([]map[string]string) { subscriber := &pb.Subscriber{ - Name: reconciler_entity.Name, - StatusMessage: deployment_entity.StatusMessage, - } - if deployment_entity.Status == "success" { - deployment_state.TotalSucceededSubscribers++ - deployment_state.SucceededSubscribers = append(deployment_state.SucceededSubscribers, subscriber) - } - if deployment_entity.Status == "failure" { - deployment_state.TotalFailedSubscribers++ - deployment_state.FailedSubscribers = append(deployment_state.FailedSubscribers, subscriber) - } - if deployment_entity.Status == "in_progress" { - deployment_state.TotalInProgressSubscribers++ - deployment_state.InProgressSubscribers = append(deployment_state.InProgressSubscribers, subscriber) + Name: deployment["name"], + StatusMessage: deployment["status_message"], } - break + deployment_state.FailedSubscribers = append(deployment_state.FailedSubscribers, subscriber) } } @@ -453,25 +455,65 @@ func readConfigValuesFromEnv() { } +func getDeploymentStateHandler(c *gin.Context) { + var req pb.DeploymentStateRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(400, gin.H{"error": err.Error()}) + return + } + + resp, err := server.GetDeploymentState(c, &req) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + c.JSON(200, resp) + +} + func main() { flag.Parse() readConfigValuesFromEnv() - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", portInt)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } + // lis, err := net.Listen("tcp", fmt.Sprintf(":%d", portInt)) + // if err != nil { + // log.Fatalf("failed to listen: %v", err) + // } grpcServer := grpc.NewServer() grpc_health_v1.RegisterHealthServer(grpcServer, health.NewServer()) dbClient := db.NewPostgresClient(*postgresHost, postgresPortInt, *postgresUser, *postgresPassword, *postgresDbName, *postgresSslmode) - pb.RegisterStorageApiServer(grpcServer, newStorageApiServer(dbClient)) + server = newStorageApiServer(dbClient) + pb.RegisterStorageApiServer(grpcServer, server) + router := gin.Default() + + router.POST("/deployment_state", getDeploymentStateHandler) + //log starting the server log.Printf("Starting server on port %d", portInt) - err = grpcServer.Serve(lis) - if err != nil { + // err = grpcServer.Serve(lis) + // if err != nil { + // log.Fatalf("failed to serve: %v", err) + // } + + httpServer := &http.Server{ + Addr: fmt.Sprintf(":%d", portInt), + Handler: h2c.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Detect if it's a gRPC request or HTTP request + if r.ProtoMajor == 2 && r.Header.Get("content-type") == "application/grpc" { + grpcServer.ServeHTTP(w, r) + } else { + router.ServeHTTP(w, r) + } + }), &http2.Server{}), + } + + log.Println("Serving gRPC and HTTP on :50051") + + if err := httpServer.ListenAndServe(); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/storage/go.mod b/storage/go.mod index 2452c02..562376c 100644 --- a/storage/go.mod +++ b/storage/go.mod @@ -5,13 +5,36 @@ go 1.19 require ( github.com/lib/pq v1.10.9 google.golang.org/grpc v1.56.3 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.1 ) require ( + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/storage/go.sum b/storage/go.sum index de5a487..de71cd5 100644 --- a/storage/go.sum +++ b/storage/go.sum @@ -1,16 +1,87 @@ +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= @@ -20,3 +91,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/storage/postgres/deployment.go b/storage/postgres/deployment.go index dbe1959..a442f07 100644 --- a/storage/postgres/deployment.go +++ b/storage/postgres/deployment.go @@ -44,26 +44,3 @@ func (d *Deployment) getByNaturalKey(conn *sql.DB) (Entity, error) { } return d, nil } - -// GetByReconcilerId -var _ QueryFunc = GetByReconcilerId - -func GetByReconcilerId(conn *sql.DB, args ...interface{}) ([]Entity, error) { - - rows, err := conn.Query(`SELECT id, gitops_commit_id, reconciler_id, status, status_message FROM deployment WHERE reconciler_id=$1 order by created_on desc`, args[0]) - if err != nil { - return nil, err - } - defer rows.Close() - - var deployments []Entity - for rows.Next() { - var d Deployment - err := rows.Scan(&d.Id, &d.GitopsCommitId, &d.ReconcilerId, &d.Status, &d.StatusMessage) - if err != nil { - return nil, err - } - deployments = append(deployments, &d) - } - return deployments, nil -} diff --git a/storage/postgres/postgres_client.go b/storage/postgres/postgres_client.go index aa76c18..c0b115c 100644 --- a/storage/postgres/postgres_client.go +++ b/storage/postgres/postgres_client.go @@ -15,13 +15,13 @@ type Entity interface { getByNaturalKey(conn *sql.DB) (Entity, error) } -type QueryFunc func(conn *sql.DB, args ...interface{}) ([]Entity, error) +type QueryFunc func(conn *sql.DB, args ...interface{}) (interface{}, error) type DBClient interface { Update(ctx context.Context, enity Entity) (Entity, error) Get(ctx context.Context, enity Entity) (Entity, error) GetByNaturalKey(ctx context.Context, enity Entity) (Entity, error) - Query(ctx context.Context, query QueryFunc, args ...interface{}) ([]Entity, error) + Query(ctx context.Context, query QueryFunc, args ...interface{}) (interface{}, error) } type postgresClient struct { @@ -87,7 +87,7 @@ func (c *postgresClient) GetByNaturalKey(ctx context.Context, entity Entity) (En return entity, nil } -func (c *postgresClient) Query(ctx context.Context, query QueryFunc, args ...interface{}) ([]Entity, error) { +func (c *postgresClient) Query(ctx context.Context, query QueryFunc, args ...interface{}) (interface{}, error) { conn, err := c.getConnection() if err != nil { return nil, err diff --git a/storage/postgres/reconciler.go b/storage/postgres/reconciler.go index bd91fc3..3e95f7e 100644 --- a/storage/postgres/reconciler.go +++ b/storage/postgres/reconciler.go @@ -1,6 +1,8 @@ package postgres -import "database/sql" +import ( + "database/sql" +) type Reconciler struct { Id int @@ -13,6 +15,12 @@ type Reconciler struct { ManifestsEndpoint string } +type StatusStats struct { + Success int32 + Failed int32 + InProgress int32 +} + // make sure that the Host implements the Entity interface var _ Entity = (*Reconciler)(nil) @@ -52,23 +60,70 @@ func (r *Reconciler) getByNaturalKey(conn *sql.DB) (Entity, error) { return r, nil } -var _ QueryFunc = GetByManifestsEndpoint +var _ QueryFunc = CountByManifestsEndpoint + +func CountByManifestsEndpoint(conn *sql.DB, args ...interface{}) (interface{}, error) { + var count int32 + + manifest_endpoint := args[0].(string) + err := conn.QueryRow(`SELECT count(1) FROM reconciler WHERE manifests_endpoint like $1`, manifest_endpoint).Scan(&count) + if err != nil { + return nil, err + } + return count, nil +} + +var _ QueryFunc = CountByStatuses + +func CountByStatuses(conn *sql.DB, args ...interface{}) (interface{}, error) { -func GetByManifestsEndpoint(conn *sql.DB, args ...interface{}) ([]Entity, error) { + status := StatusStats{} - rows, err := conn.Query(`SELECT id, name, host_id, description, reconciler_type, labels, manifests_storage_type, manifests_endpoint FROM reconciler WHERE manifests_endpoint like '$1%'`, args[0]) + manifest_endpoint := args[0].(string) + gitops_commit_id := args[1].(string) + err := conn.QueryRow(`SELECT count(case when status='success' then 1 end) as success, + count(case when status='failure' then 1 end) as failed, + count(case when status='in_progress' then 1 end) as in_progress + FROM reconciler r, deployment d + WHERE r.id = d.reconciler_id and + r.manifests_endpoint like $1 and + d.gitops_commit_id = $2`, + manifest_endpoint, gitops_commit_id).Scan(&status.Success, &status.Failed, &status.InProgress) + if err != nil { + return nil, err + } + return status, nil +} + +var _ QueryFunc = GetByStatus + +func GetByStatus(conn *sql.DB, args ...interface{}) (interface{}, error) { + manifest_endpoint := args[0].(string) + gitops_commit_id := args[1].(string) + status := args[2].(string) + reconcilers := make([]map[string]string, 0) + + rows, err := conn.Query(`SELECT r.name, d.status_message + FROM reconciler r, deployment d + WHERE r.id = d.reconciler_id and + r.manifests_endpoint like $1 and + d.gitops_commit_id = $2 and + d.status = $3`, + manifest_endpoint, gitops_commit_id, status) if err != nil { return nil, err } defer rows.Close() - var reconcilers []Entity for rows.Next() { - var reconciler Reconciler - err := rows.Scan(&reconciler.Id, &reconciler.Name, &reconciler.HostId, &reconciler.Description, &reconciler.ReconcilerType, &reconciler.Labels, &reconciler.ManifestsStorageType, &reconciler.ManifestsEndpoint) + var reconciler_name string + var reconciler_status_message string + + err := rows.Scan(&reconciler_name, &reconciler_status_message) if err != nil { return nil, err } - reconcilers = append(reconcilers, &reconciler) + reconcilers = append(reconcilers, map[string]string{"name": reconciler_name, "status_message": reconciler_status_message}) } return reconcilers, nil + }