remote build is now a streaming endpoint that sends the logs every 1s

remotebuild/main.go

@@ -63,10 +63,11 @@ func main() {
 func (s *Server) Build(
 	ctx context.Context,
 	req *connect.Request[pbbuild.BuildRequest],
-) (*connect.Response[pbbuild.BuildResponse], error) {
+	stream *connect.ServerStream[pbbuild.BuildResponse],
+) error {
 	tempDir, err := os.MkdirTemp(os.TempDir(), "remotebuild")
 	if err != nil {
-		return nil, fmt.Errorf("creating temp dir: %w", err)
+		return fmt.Errorf("creating temp dir: %w", err)
 	}
 
 	if os.Getenv("GENERATOR_KEEP_FILES") != "true" {
@@ -83,21 +84,22 @@ func (s *Server) Build(
 	s.logger.Debug("source code size", "size", len(req.Msg.SourceCode))
 	err = unzip(req.Msg.SourceCode, tempDir)
 	if err != nil {
-		return nil, fmt.Errorf("unzipping: %w", err)
+		return fmt.Errorf("unzipping: %w", err)
 	}
 
 	folders, err := os.ReadDir(tempDir)
 	if err != nil {
-		return nil, fmt.Errorf("reading temp dir: %w", err)
+		return fmt.Errorf("reading temp dir: %w", err)
 	}
 
 	if len(folders) != 1 {
-		return nil, fmt.Errorf("expected exactly one folder in temp dir, got %d", len(folders))
+		return fmt.Errorf("expected exactly one folder in temp dir, got %d", len(folders))
 	}
 
 	// here inside the folder, it will contain the name of the folder that the files were unzipped into
 	workingDir := filepath.Join(tempDir, folders[0].Name())
 
+	// Check to add this in the docker file and make it work as expected
 	content := dedent.Dedent(`
 		rustup target add wasm32-unknown-unknown
 		make package
@@ -106,39 +108,72 @@ func (s *Server) Build(
 	// here we need to write a run.sh script to allow /bin/sh to run multiple commands
 	// from the Makefile. If not, it will always only run the first command (make protogen)
 	if err := os.WriteFile(filepath.Join(workingDir, "run.sh"), []byte(content), 0755); err != nil {
-		return nil, fmt.Errorf("writing run.sh: %w", err)
+		return fmt.Errorf("writing run.sh: %w", err)
 	}
 
-	cmd := exec.Command("/bin/sh", "-c", "./run.sh")
+	cmd := exec.CommandContext(ctx, "/bin/sh", "-c", "./run.sh")
 
 	// Setup environmental variables for the command run
 	// Also add in any environmental variables passed in the request
 	cmd.Env = append(req.Msg.Env, os.Environ()...)
 
 	cmd.Dir = workingDir
+
 	stdoutBuf := &bytes.Buffer{}
 	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutBuf)
 	cmd.Stderr = io.MultiWriter(os.Stderr, stdoutBuf)
+
+	progressLogsOffset := 0
+
+	ctx, cancelProgressSender := context.WithCancel(ctx)
+	go func() {
+		timer := time.NewTicker(1 * time.Second)
+		defer timer.Stop()
+
+		for {
+			select {
+			case <-timer.C:
+				to := len(stdoutBuf.Bytes())
+				progressLogs := stdoutBuf.Bytes()[progressLogsOffset:to]
+				// todo keep sending progress logs
+				stream.Send(&pbbuild.BuildResponse{
+					Logs: string(progressLogs),
+				})
+				progressLogsOffset = to
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
 	err = cmd.Run()
+	cancelProgressSender()
+
 	if err != nil {
-		return connect.NewResponse(&pbbuild.BuildResponse{
+		stream.Send(&pbbuild.BuildResponse{
 			Error: err.Error(),
-			Logs:  stdoutBuf.String(),
-		}), nil
+			// send the rest of the logs that have not been sent yet
+			Logs: string(stdoutBuf.Bytes()[progressLogsOffset:]),
+		})
+		return fmt.Errorf("running build command: %w", err)
 	}
 
-	s.logger.Debug("make package", "output", stdoutBuf.String())
-
 	artifacts, err := s.collectArtifacts(workingDir, "substreams.spkg")
 	if err != nil {
-		return nil, fmt.Errorf("collecting artifacts: %w", err)
+		stream.Send(&pbbuild.BuildResponse{
+			Error: err.Error(),
+			// no logs to send here as we have an error which is unreleated to the build
+		})
+		return fmt.Errorf("collecting artifacts: %w", err)
 	}
 
-	resp := &pbbuild.BuildResponse{
+	stream.Send(&pbbuild.BuildResponse{
 		Artifacts: artifacts,
-	}
+		// send the rest of the logs that have not been sent yet
+		Logs: string(stdoutBuf.Bytes()[progressLogsOffset:]),
+	})
 
-	return connect.NewResponse(resp), nil
+	return nil
 }
 
 func (s *Server) collectArtifacts(dir string, pattern string) (out []*pbbuild.BuildResponse_BuildArtifact, err error) {