Skip to content

Commit

Permalink
remove stale update code and feature flag
Browse files Browse the repository at this point in the history 
Signed-off-by: Jonathan Holloway <[email protected]>
  • Loading branch information
@loadtheaccumulator
loadtheaccumulator committed Feb 19, 2025
1 parent 6f34293 commit 0223d8c
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 967 deletions.
296 changes: 6 additions & 290 deletions pkg/services/repobuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/redhatinsights/edge-api/pkg/db"
"github.com/redhatinsights/edge-api/pkg/models"
"github.com/redhatinsights/edge-api/pkg/services/repostore"
feature "github.com/redhatinsights/edge-api/unleash/features"

"github.com/cavaliergopher/grab/v3"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -92,267 +91,13 @@ func (rb *RepoBuilder) BuildUpdateRepo(ctx context.Context, id uint) (*models.Up
return nil, errors.New("repo unavailable")
}

// Skipping this process if the feature flag is enabled.
// Without static-deltas, this just downloads a commit repo and re-uploads as an update repo.
// This will retain the original commit repo URL as the update URL.
// (e.g., for a 2GB commit, this saves 4GB+ in traffic and local disk space on the pod)
if !feature.SkipUpdateRepo.IsEnabled() {
rb.log.WithField("update_transaction", update).Info("UPGRADE: update repo build started")

fromCommitHash := "undefined"
fromCommit := &models.StaticDeltaCommit{
Rev: fromCommitHash,
}

oldCommitCount := len(update.OldCommits)
if oldCommitCount > 0 {
fromCommit.Rev = update.OldCommits[0].OSTreeCommit
}

toCommit := &models.StaticDeltaCommit{
Rev: update.Commit.OSTreeCommit,
}

// FIXME: replace with new struct
rb.log.WithFields(log.Fields{
"old_commit_count": oldCommitCount,
"from_commit_hash": fromCommit.Rev,
"to_commit_hash": toCommit.Rev,
}).Info("UPGRADE: static delta commit to and from is set")

staticDeltaState := &models.StaticDeltaState{
Name: models.GetStaticDeltaName(fromCommit.Rev, toCommit.Rev),
OrgID: update.OrgID,
Status: models.StaticDeltaStatusDownloading,
}

if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
return nil, errors.New("error saving static delta state")
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// create the local FQ path
cfg := config.Get()
path := filepath.Clean(filepath.Join(cfg.RepoTempPath, "upd/", strconv.FormatUint(uint64(update.ID), 10)))
err := os.MkdirAll(path, os.FileMode(0755))
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error creating update path")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}
return nil, err
}
rb.log.WithField("path", path).Info("UPGRADE: local update path set")

// change to FQ path directory
err = os.Chdir(path)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error changing directory to update path")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, err
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// download the commit tarfile
tarFileName, err := rb.CommitTarDownload(update.Commit, path)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error downloading tar")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, fmt.Errorf("error download repo repo :: %s", err.Error())
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// extract the tarfile to local FQ path
err = rb.CommitTarExtract(update.Commit, tarFileName, path)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error extracting tar")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, fmt.Errorf("error extracting repo :: %s", err.Error())
}
rb.log.WithFields(log.Fields{
"updateID": update.ID,
"OldCommits": len(update.OldCommits)}).
Info("Old commits found to this commit")

err = rb.CommitTarDelete(tarFileName)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error deleting commit tarfile")
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// create local FQ path for static delta work
stagePath := filepath.Clean(filepath.Join(path, "staging"))
err = os.MkdirAll(stagePath, os.FileMode(0755))
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error making dir")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, fmt.Errorf("error mkdir :: %s", err.Error())
}
rb.log.WithField("stage_path", stagePath).Info("UPGRADE: static delta stage path set")

err = os.Chdir(stagePath)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error changing dir")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, fmt.Errorf("error chdir :: %s", err.Error())
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// If there are any old commits, we need to download them all to be merged
// into the update commit repo
for _, commit := range update.OldCommits {
rb.log.WithFields(log.Fields{
"updateID": update.ID,
"commit.OSTreeCommit": commit.OSTreeCommit,
"OldCommits": commit.ID}).
Info("Calculate diff from previous commit")
commit := commit // this will prevent implicit memory aliasing in the loop
stageCommitPath := filepath.Clean(filepath.Join(stagePath, commit.OSTreeCommit))
tarFileName, err := rb.CommitTarDownload(&commit, stageCommitPath)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error downloading tar")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, fmt.Errorf("error Upload repo repo :: %s", err.Error())
}

rb.log.WithField("update_transaction", update).Info("UPGRADE: to_commit repo tarfile downloaded")

err = rb.CommitTarExtract(update.Commit, tarFileName, stageCommitPath)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error extracting repo")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, err
}

err = rb.CommitTarDelete(tarFileName)
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error deleting commit tarfile")
}

rb.log.WithField("update_transaction", update).Info("UPGRADE: to_commit repo tarfile extracted")

staticDeltaState.Status = models.StaticDeltaStatusGenerating
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")

return nil, errors.New("Error saving static delta state")
}

rb.log.WithField("update_transaction", update).Info("UPGRADE: updating repo with static delta and summary")
err = rb.RepoPullLocalStaticDeltas(update.Commit, &commit, filepath.Clean(filepath.Join(path, "repo")),
filepath.Clean(filepath.Join(stageCommitPath, "repo")))
if err != nil {
rb.log.WithField("error", err.Error()).Error("Error pulling static deltas")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, err
}
rb.log.WithField("update_transaction", update).Info("UPGRADE: updating repo complete")
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

// Once all the old commits have been pulled into the update commit's repo
// and has static deltas generated, then we don't need the old commits
// anymore.
err = os.RemoveAll(stagePath)
if err != nil {
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, err
}

// NOTE: This relies on the file path being cfg.RepoTempPath/models.Repo.ID/

rb.log.Info("Upload repo")

staticDeltaState.Status = models.StaticDeltaStatusUploading
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")

return nil, errors.New("Error saving static delta state")
}

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")

repoURL, err := rb.FilesService.GetUploader().UploadRepo(filepath.Clean(filepath.Join(path, "repo")), strconv.FormatUint(uint64(update.ID), 10), "private")
if err != nil {
rb.log.WithField("error", err.Error()).Error("error occurred while uploading repo")
staticDeltaState.Status = models.StaticDeltaStatusError
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")
}

return nil, err
}
rb.log.Info("Finished uploading repo")

staticDeltaState.Status = models.StaticDeltaStatusReady
staticDeltaState.URL = repoURL
if saveErr := staticDeltaState.Save(rb.log); saveErr != nil {
rb.log.WithField("error", saveErr.Error()).Error("error saving static delta state")

return nil, errors.New("Error saving static delta state")
}

update.Repo.URL = repoURL
rb.log.WithField("update_transaction", update).Info("UPGRADE: point update to new update repo")

rb.log.WithField("static_delta_state", staticDeltaState).Info("UPGRADE: static delta state")
}

// FIXME: (remove) moved this logic to the calling function to avoid extra DB lookups when static delta is not generated
// remove when tested successfully
if feature.SkipUpdateRepo.IsEnabled() {
// grab the original commit URL
updateCommit, err := rb.commitService.GetCommitByID(update.CommitID, update.OrgID)
if err != nil {
return nil, err
}
update.Repo.URL = updateCommit.Repo.DistributionURL(ctx)
rb.log.WithField("update_transaction", update).Info("UPGRADE: point update to commit repo")
// grab the original commit URL
updateCommit, err := rb.commitService.GetCommitByID(update.CommitID, update.OrgID)
if err != nil {
return nil, err
}
update.Repo.URL = updateCommit.Repo.ContentURL(ctx)
rb.log.WithField("update_transaction", update).Info("UPGRADE: point update to commit repo")

rb.log.WithField("repo", update.Repo.DistributionURL(ctx)).Info("Update repo URL")
update.Repo.Status = models.RepoStatusSuccess
Expand Down Expand Up @@ -593,35 +338,6 @@ func (rb *RepoBuilder) CommitTarExtract(c *models.Commit, tarFileName string, de

rb.log.WithField("filepath", tarFileName).Debug("Unpacking tarball finished")

// Commenting this Block, as failing, and seems never was working, fixing this block will create a new commit with
// a new checksum that need to be recorded back to the current commit, this will also need to change the logic of
// the caller function of this function, this need more discussion, a bug has been created.
/*
var cmd *exec.Cmd
if c.OSTreeRef == "" {
refs := config.DistributionsRefs[config.DefaultDistribution]
cmd = &exec.Cmd{
Path: "/usr/bin/ostree",
Args: []string{
"--repo", "./repo", "commit", refs, "--add-metadata-string", fmt.Sprintf("version=%s.%d", c.BuildDate, c.BuildNumber),
},
}
} else {
cmd = &exec.Cmd{
Path: "/usr/bin/ostree",
Args: []string{
"--repo", "./repo", "commit", c.OSTreeRef, "--add-metadata-string", fmt.Sprintf("version=%s.%d", c.BuildDate, c.BuildNumber),
},
}
}
err = cmd.Run()
if err != nil {
rb.log.WithFields(log.Fields{
"error": err.Error(),
"command": fmt.Sprintf("%s %s %s %s %s %s %s", "ostree", "--repo", "./repo", "commit", c.OSTreeRef, "--add-metadata-string", fmt.Sprintf("version=%s.%d", c.BuildDate, c.BuildNumber)),
}).Error("OSTree command failed")
}
*/
return nil
}

Expand Down
Loading

0 comments on commit 0223d8c

Please sign in to comment.