Skip to content

Commit

Permalink
fix the 'skipping run' on 'existing exec output for output_modules': …
Browse files Browse the repository at this point in the history
…only use it if there are no stores to write. race condition with other jobs could lead to that condition
  • Loading branch information
sduchesneau committed Jan 7, 2025
1 parent d224772 commit 6227346
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions service/tier2.go
Original file line number Diff line number Diff line change
Expand Up @@ -691,11 +691,6 @@ func GetExecutionPlan(
}
existingExecOuts[name] = file

if runningLastStage && name == outputModule {
logger.Info("found existing exec output for output_module, skipping run", zap.String("output_module", name))
return nil, nil
}

case pbsubstreams.ModuleKindStore:
file, readErr := c.ReadFile(ctx, &block.Range{StartBlock: moduleStartBlock, ExclusiveEndBlock: stopBlock})
if readErr != nil {
Expand Down Expand Up @@ -725,6 +720,11 @@ func GetExecutionPlan(

}

if runningLastStage && len(storesToWrite) == 0 && existingExecOuts[outputModule] != nil {
logger.Info("found existing exec output for output_module and no stores to produce, skipping run", zap.String("output_module", outputModule))
return nil, nil
}

for name, module := range requiredModules {
if _, exists := existingExecOuts[name]; exists {
continue // for stores that need to be run for the partials, but already have cached execution outputs
Expand Down

0 comments on commit 6227346

Please sign in to comment.