Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update for handle failed deployment case #501

Merged
merged 13 commits into from
Jul 5, 2024
16 changes: 13 additions & 3 deletions src/cmd/cli/command/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/DefangLabs/defang/src/pkg/scope"
"github.com/DefangLabs/defang/src/pkg/term"
"github.com/DefangLabs/defang/src/pkg/types"
defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
"github.com/aws/smithy-go"
"github.com/bufbuild/connect-go"
"github.com/spf13/cobra"
Expand All @@ -30,6 +31,9 @@ import (
const DEFANG_PORTAL_HOST = "portal.defang.dev"
const SERVICE_PORTAL_URL = "https://" + DEFANG_PORTAL_HOST + "/service"

var ErrFailedToReachStartedState = errors.New("failed to reach STARTED state")
var ErrDeploymentFailed = errors.New("deployment failed")

const authNeeded = "auth-needed" // annotation to indicate that a command needs authorization
var authNeededAnnotation = map[string]string{authNeeded: ""}

Expand Down Expand Up @@ -840,16 +844,22 @@ var composeUpCmd = &cobra.Command{
}
}
}()
if err := waitServiceStatus(ctx, cli.ServiceStarted, serviceInfos); err != nil && !errors.Is(err, context.Canceled) {
if !errors.Is(err, cli.ErrDryRun) && !errors.As(err, new(cliClient.ErrNotImplemented)) {

if err := waitServiceState(ctx, defangv1.ServiceState_SERVICE_COMPLETED, serviceInfos); err != nil && !errors.Is(err, context.Canceled) {
if errors.Is(err, ErrDeploymentFailed) {
term.Warn("Deployment FAILED. Service(s) not running.")
nullfunc marked this conversation as resolved.
Show resolved Hide resolved
return err
} else {
term.Warnf("failed to wait for service status: %v", err)
}
wg.Wait() // Wait until ctrl+c is pressed
lionello marked this conversation as resolved.
Show resolved Hide resolved

wg.Wait() // Wait for tail ctrl + c
}
cancelTail()
wg.Wait() // Wait for tail to finish

printEndpoints(serviceInfos)

term.Info("Done.")
return nil
},
Expand Down
8 changes: 7 additions & 1 deletion src/cmd/cli/command/deploymentinfo.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@ func printEndpoints(serviceInfos []*defangv1.ServiceInfo) {
if len(serviceInfo.Endpoints) > 0 {
andEndpoints = "and will be available at:"
}
term.Info("Service", serviceInfo.Service.Name, "is in state", serviceInfo.Status, andEndpoints)

serviceConditionText := "has status " + serviceInfo.Status
if serviceInfo.State != defangv1.ServiceState_NOT_SPECIFIED {
serviceConditionText = "is in state " + serviceInfo.State.String()
}

term.Info("Service", serviceInfo.Service.Name, serviceConditionText, andEndpoints)
for i, endpoint := range serviceInfo.Endpoints {
if serviceInfo.Service.Ports[i].Mode == defangv1.Mode_INGRESS {
endpoint = "https://" + endpoint
Expand Down
31 changes: 18 additions & 13 deletions src/cmd/cli/command/servicemonitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,17 @@ package command

import (
"context"
"fmt"

"github.com/DefangLabs/defang/src/pkg/cli"
"github.com/DefangLabs/defang/src/pkg/cli/compose"
"github.com/DefangLabs/defang/src/pkg/term"
defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
)

func waitServiceStatus(ctx context.Context, targetStatus cli.ServiceStatus, serviceInfos []*defangv1.ServiceInfo) error {
func waitServiceState(ctx context.Context, targetState defangv1.ServiceState, serviceInfos []*defangv1.ServiceInfo) error {
serviceList := []string{}
for _, serviceInfo := range serviceInfos {
serviceList = append(serviceList, serviceInfo.Service.Name)
serviceList = append(serviceList, compose.NormalizeServiceName(serviceInfo.Service.Name))
}

// set up service status subscription (non-blocking)
Expand All @@ -22,34 +22,39 @@ func waitServiceStatus(ctx context.Context, targetStatus cli.ServiceStatus, serv
return err
}

serviceStatus := make(map[string]string, len(serviceList))
serviceState := make(map[string]defangv1.ServiceState, len(serviceList))
nullfunc marked this conversation as resolved.
Show resolved Hide resolved
for _, name := range serviceList {
serviceStatus[name] = string(cli.ServiceUnknown)
serviceState[name] = defangv1.ServiceState_NOT_SPECIFIED
}

// monitor for when all services are completed to end this command
for newStatus := range subscribeServiceStatusChan {
if _, ok := serviceStatus[newStatus.Name]; !ok {
if _, ok := serviceState[newStatus.Name]; !ok {
term.Debugf("unexpected service %s update", newStatus.Name)
continue
}

serviceStatus[newStatus.Name] = newStatus.Status
// exit on detecting a FAILED state
if newStatus.State == defangv1.ServiceState_SERVICE_FAILED {
return ErrDeploymentFailed
}

serviceState[newStatus.Name] = newStatus.State

if allInStatus(targetStatus, serviceStatus) {
if allInState(targetState, serviceState) {
for _, sInfo := range serviceInfos {
sInfo.Status = string(targetStatus)
sInfo.State = targetState
}
return nil
}
}

return fmt.Errorf("service state monitoring terminated without all services reaching desired state: %s", targetStatus)
return ErrFailedToReachStartedState
}

func allInStatus(targetStatus cli.ServiceStatus, serviceStatuses map[string]string) bool {
for _, status := range serviceStatuses {
if status != string(targetStatus) {
func allInState(targetState defangv1.ServiceState, serviceStates map[string]defangv1.ServiceState) bool {
for _, state := range serviceStates {
if state != targetState {
return false
}
}
Expand Down
2 changes: 2 additions & 0 deletions src/pkg/cli/client/byoc/aws/byoc.go
Original file line number Diff line number Diff line change
Expand Up @@ -569,8 +569,10 @@ func (b *ByocAws) update(ctx context.Context, service *defangv1.Service) (*defan

si.NatIps = b.publicNatIps // TODO: even internal services use NAT now
si.Status = "UPDATE_QUEUED"
si.State = defangv1.ServiceState_UPDATE_QUEUED
if si.Service.Build != nil {
si.Status = "BUILD_QUEUED" // in SaaS, this gets overwritten by the ECS events for "kaniko"
si.State = defangv1.ServiceState_BUILD_QUEUED
}
return si, nil
}
Expand Down
13 changes: 13 additions & 0 deletions src/pkg/cli/compose/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,3 +321,16 @@ func convertPorts(ports []compose.ServicePortConfig) []*defangv1.Port {
}
return pbports
}

func ConvertServiceState(state string) defangv1.ServiceState {
switch strings.ToUpper(state) {
default:
return defangv1.ServiceState_NOT_SPECIFIED
case "IN_PROGRESS", "STARTING":
return defangv1.ServiceState_SERVICE_PENDING
case "COMPLETED":
return defangv1.ServiceState_SERVICE_COMPLETED
case "FAILED":
return defangv1.ServiceState_SERVICE_FAILED
}
}
32 changes: 12 additions & 20 deletions src/pkg/cli/subscribe.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,14 @@ import (
type SubscribeServiceStatus struct {
Name string
Status string
State defangv1.ServiceState
}

func Subscribe(ctx context.Context, client client.Client, services []string) (<-chan SubscribeServiceStatus, error) {
if len(services) == 0 {
return nil, fmt.Errorf("no services specified")
}

normalizedServiceNameToServiceName := make(map[string]string, len(services))

for i, service := range services {
services[i] = compose.NormalizeServiceName(service)
normalizedServiceNameToServiceName[services[i]] = service
}

statusChan := make(chan SubscribeServiceStatus, len(services))
if DoDryRun {
defer close(statusChan)
Expand Down Expand Up @@ -61,23 +55,21 @@ func Subscribe(ctx context.Context, client client.Client, services []string) (<-
continue
}

servInfo := msg.GetService()
if servInfo == nil || servInfo.Service == nil {
continue
subStatus := SubscribeServiceStatus{
Name: msg.GetName(),
Status: msg.GetStatus(),
State: msg.GetState(),
}

serviceName, ok := normalizedServiceNameToServiceName[servInfo.Service.Name]
if !ok {
term.Debugf("Unknown service %s in subscribe response\n", servInfo.Service.Name)
continue
}
status := SubscribeServiceStatus{
Name: serviceName,
Status: servInfo.Status,
servInfo := msg.GetService()
if subStatus.Name == "" && (servInfo != nil && servInfo.Service != nil) {
subStatus.Name = servInfo.Service.Name
subStatus.Status = servInfo.Status
subStatus.State = compose.ConvertServiceState(servInfo.Status)
}

statusChan <- status
term.Debugf("service %s with status %s\n", serviceName, servInfo.Status)
statusChan <- subStatus
term.Debugf("service %s with state ( %s ) and status: %s\n", subStatus.Name, subStatus.State.String(), subStatus.Status)
}
}()

Expand Down
26 changes: 17 additions & 9 deletions src/pkg/cli/tail.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,23 @@ var (
type ServiceStatus string

const (
nullfunc marked this conversation as resolved.
Show resolved Hide resolved
ServiceDeploymentStarting ServiceStatus = "STARTING"
ServiceDeploymentInProgress ServiceStatus = "IN_PROGRESS"
ServiceStarted ServiceStatus = "COMPLETED"
ServiceStopping ServiceStatus = "STOPPING"
ServiceStopped ServiceStatus = "STOPPED"
ServiceDeactivating ServiceStatus = "DEACTIVATING"
ServiceDeprovisioning ServiceStatus = "DEPROVISIONING"
ServiceFailed ServiceStatus = "FAILED"
ServiceUnknown ServiceStatus = "UNKNOWN"
ServiceUnspecified ServiceStatus = "UNSPECIFIED"

// build states
ServiceBuildQueued ServiceStatus = "BUILD_QUEUED"
ServiceBuildProvisioning ServiceStatus = "BUILD_PROVISIONING"
ServiceBuildPending ServiceStatus = "BUILD_PENDING"
ServiceBuildActivating ServiceStatus = "BUILD_ACTIVATING"
ServiceBuildRunning ServiceStatus = "BUILD_RUNNING"
ServiceBuildDeactivating ServiceStatus = "BUILD_DEACTIVATING" // build completed

// update states
ServiceUpdateQueued ServiceStatus = "UPDATE_QUEUED" // queued for deployment

// deplpyment states
ServicePending ServiceStatus = "PENDING"
ServiceCompleted ServiceStatus = "COMPLETED"
ServiceFailed ServiceStatus = "FAILED"
)

type EndLogConditional struct {
Expand Down
Loading