Image import from tar archives

Makefile

@@ -2,6 +2,7 @@ GOSRC = ./../..
 EUDATSRC = ./..
 WEBUI = frontend/webui
 INTERNALSERVICES = services/_internal
+GOFLAGS=-ldflags -s
 
 build: dependencies webui frontend containers backend
 
@@ -10,14 +11,14 @@ webui:
 
 containers:
 	(cd $(INTERNALSERVICES)/volume-stage-in && docker build -t volume-stage-in .)
-	(cd $(INTERNALSERVICES)/volume-filelist && GOOS=linux GOARCH=amd64 go build && docker build -t volume-filelist .)
+	(cd $(INTERNALSERVICES)/volume-filelist && GOOS=linux GOARCH=amd64 go build $(GOFLAGS) && docker build -t volume-filelist .)
 	(cd $(INTERNALSERVICES)/copy-from-volume && docker build -t copy-from-volume .)
 
 backend:
 	$(GOPATH)/bin/golint ./...
 	go vet ./...
-	go test ./...
-	go build ./...
+	go test $(GOFLAGS) ./...
+	go build $(GOFLAGS) ./...
 
 dependencies: $(WEBUI)/node_modules $(GOSRC)/golang/lint/golint $(GOSRC)/fsouza/go-dockerclient $(GOSRC)/gorilla/mux $(GOSRC)/pborman/uuid $(GOSRC)/gopkg.in/gorp.v1 $(GOSRC)github.com/mattn/go-sqlite3
 
@@ -46,6 +47,6 @@ webui_dev_server:
 	(cd $(WEBUI) && node_modules/webpack-dev-server/bin/webpack-dev-server.js --config webpack.config.devel.js)
 
 run_gef:
-	(cd backend-docker && go run main.go)
+	(cd backend-docker && go run $(GOFLAGS) main.go)
 
 .PHONY: build dependencies webui frontend backend webui_dev_server run_frontend run_backend clean

backend-docker/db/db.go

@@ -16,7 +16,6 @@ const SQLiteDataBasePath = "gef_db.bin"
 // Db is used to keep DbMap
 type Db struct{ gorp.DbMap }
 
-
 // JobTable stores the information about a service execution (used to store data in a database)
 type JobTable struct {
 	ID           string

backend-docker/pier/internal/dckr/dckr.go

@@ -15,6 +15,8 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 
+	"encoding/json"
+
 	"github.com/EUDAT-GEF/GEF/backend-docker/def"
 )
 
@@ -520,3 +522,85 @@ func (c Client) UploadFile2Container(containerID, srcPath string, dstPath string
 	err = c.c.UploadToContainer(containerID, opts)
 	return err
 }
+
+func ExtractImageIDFromTar(imageFilePath string) (string, error) {
+	type Manifest struct {
+		Config   string
+		RepoTags []string
+		Layers   []string
+	}
+
+	var foundID string
+
+	tarImage, err := os.Open(imageFilePath)
+	if err != nil {
+		return foundID, err
+	}
+
+	tarBallReader := tar.NewReader(tarImage)
+	for {
+		hdr, err := tarBallReader.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return foundID, def.Err(err, "reading tarball failed")
+		}
+
+		if strings.ToLower(hdr.Name) == "manifest.json" {
+			var manifestContent []Manifest
+			buf := new(bytes.Buffer)
+			buf.ReadFrom(tarBallReader)
+
+			err = json.Unmarshal(buf.Bytes(), &manifestContent)
+			if err != nil {
+				if err != nil {
+					return foundID, def.Err(err, "cannot read manifest.json from the image")
+				}
+			}
+
+			for _, value := range manifestContent {
+				foundID := strings.Replace(value.Config, ".json", "", 1)
+				if len(foundID) > 1 {
+					return foundID, nil
+				}
+			}
+		}
+	}
+
+	return foundID, def.Err(err, "could not retrieve image information")
+}
+
+func (c *Client) ImportImageFromTar(imageFilePath string) (ImageID, error) {
+	var id string
+
+	id, err := ExtractImageIDFromTar(imageFilePath)
+	if err != nil {
+		return ImageID(id), err
+	}
+
+	tar, err := os.Open(imageFilePath)
+	if err != nil {
+		return ImageID(id), err
+	}
+	defer tar.Close()
+
+	opts := docker.LoadImageOptions{
+		InputStream: tar,
+	}
+
+	err = c.c.LoadImage(opts)
+
+	return ImageID(id), err
+}
+
+func (c *Client) TagImage(id string, repo string, tag string) error {
+	opts := docker.TagImageOptions{
+		Repo:  repo,
+		Tag:   tag,
+		Force: true,
+	}
+
+	return c.c.TagImage(id, opts)
+}

backend-docker/pier/pier.go

@@ -41,6 +41,7 @@ func NewPier(cfgList []def.DockerConfig, tmpDir string, dataBase *db.Db) (*Pier,
 		db:     dataBase,
 		tmpDir: tmpDir,
 	}
+
 	return &pier, nil
 }
 
@@ -79,6 +80,7 @@ func (p *Pier) RunService(service db.Service, inputPID string) (db.Job, error) {
 
 // runJob runs a job
 func (p *Pier) runJob(job *db.Job, service db.Service, inputPID string) {
+
 	p.db.SetJobState(job.ID, db.JobState{"", "Creating a new input volume", -1})
 	inputVolume, err := p.docker.NewVolume()
 	if err != nil {
@@ -98,7 +100,6 @@ func (p *Pier) runJob(job *db.Job, service db.Service, inputPID string) {
 			errMsg = err.Error()
 		}
 		p.db.AddJobTask(job.ID, "Data staging", string(containerID), errMsg, exitCode, consoleOutput)
-		//fmt.Println(containerID, consoleOutput)
 
 		log.Println("  staging ended: ", exitCode, ", error: ", err)
 		if err != nil {
@@ -131,7 +132,6 @@ func (p *Pier) runJob(job *db.Job, service db.Service, inputPID string) {
 			errMsg = err.Error()
 		}
 		p.db.AddJobTask(job.ID, "Service execution", string(containerID), errMsg, exitCode, consoleOutput)
-		//fmt.Println(containerID, consoleOutput)
 
 		log.Println("  job ended: ", exitCode, ", error: ", err)
 		if err != nil {
@@ -191,21 +191,52 @@ func (p *Pier) PopulateServiceTable() error {
 				img, err := p.docker.BuildImage(filepath.Join(servicesFolder, f.Name()))
 
 				if err != nil {
-					log.Print("failed to create a service")
+					log.Print("failed to create a service: ", err)
 				} else {
 					log.Print("service has been created")
-					error := p.db.AddService(NewServiceFromImage(img))
-					if error != nil {
-						log.Print(error)
+
+					err = p.docker.TagImage(string(img.ID), f.Name(), "latest")
+					if err != nil {
+						log.Print("could not tag the service")
+					}
+
+					img, err = p.docker.InspectImage(img.ID)
+					if err != nil {
+						log.Print("failed to inspect the image: ", err)
+					}
+
+					err = p.db.AddService(NewServiceFromImage(img))
+					if err != nil {
+						log.Print("failed to add the service to the database: ", err)
 					}
 				}
 			}
 		}
 	}
-
 	return nil
 }
 
+func (p *Pier) ImportImage(imageFilePath string) (db.Service, error) {
+	imageID, err := p.docker.ImportImageFromTar(imageFilePath)
+	if err != nil {
+		return db.Service{}, def.Err(err, "docker ImportImage failed")
+	}
+
+	image, err := p.docker.InspectImage(imageID)
+
+	if err != nil {
+		return db.Service{}, err
+	}
+
+	service := NewServiceFromImage(image)
+	err = p.db.AddService(service)
+	if err != nil {
+		return db.Service{}, def.Err(err, "could not add a new service to the database")
+	}
+
+	return service, nil
+}
+
 // NewServiceFromImage extracts metadata and creates a valid GEF service
 func NewServiceFromImage(image dckr.Image) db.Service {
 	srv := db.Service{

backend-docker/server/api.go

@@ -144,6 +144,11 @@ func (s *Server) buildImageHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	var service db.Service
+
+	foundImageFileName := ""
+	tarFileFound := false
+	dockerFileFound := false
 	for {
 		part, err := mr.NextPart()
 		if err == io.EOF {
@@ -152,6 +157,7 @@ func (s *Server) buildImageHandler(w http.ResponseWriter, r *http.Request) {
 		if part.FileName() == "" {
 			continue
 		}
+
 		dst, err := os.Create(filepath.Join(buildDir, part.FileName()))
 		if err != nil {
 			Response{w}.ServerError("while creating file to save file part ", err)
@@ -163,17 +169,46 @@ func (s *Server) buildImageHandler(w http.ResponseWriter, r *http.Request) {
 			Response{w}.ServerError("while dumping file part ", err)
 			return
 		}
-	}
 
-	if _, err := os.Stat(filepath.Join(buildDir, "Dockerfile")); os.IsNotExist(err) {
-		Response{w}.ServerError("no Dockerfile to build new image ", err)
-		return
+		if strings.HasSuffix(strings.ToLower(part.FileName()), ".tar") || strings.HasSuffix(strings.ToLower(part.FileName()), ".tar.gz") {
+			tarFileFound = true
+			foundImageFileName = part.FileName()
+		}
+
+		if strings.ToLower(part.FileName()) == "dockerfile" {
+			dockerFileFound = true
+		}
+
 	}
 
-	service, err := s.pier.BuildService(buildDir)
-	if err != nil {
-		Response{w}.ServerError("build service failed: ", err)
-		return
+	// Building an image from a Dockerfile
+	if dockerFileFound {
+		if _, err := os.Stat(filepath.Join(buildDir, "Dockerfile")); os.IsNotExist(err) {
+			Response{w}.ServerError("no Dockerfile to build new image ", err)
+			return
+		}
+
+		service, err = s.pier.BuildService(buildDir)
+		if err != nil {
+			Response{w}.ServerError("build service failed: ", err)
+			return
+		}
+	} else {
+		// Importing an existing image from a tar archive
+		if tarFileFound {
+			log.Println("Docker image file has been detected, trying to import")
+			log.Println(filepath.Join(buildDir, foundImageFileName))
+			service, err = s.pier.ImportImage(filepath.Join(buildDir, foundImageFileName))
+			if err != nil {
+				Response{w}.ServerError("while importing a Docker image file ", err)
+				return
+			}
+
+			log.Println("Docker image has been imported")
+		} else {
+			Response{w}.ServerNewError("there is neither Dockerfile nor Tar archive")
+			return
+		}
 	}
 
 	Response{w}.Ok(jmap("Service", service))

frontend/webui/package.json

@@ -30,7 +30,7 @@
     "react": "^15.4.2",
     "react-bootstrap": "^0.30.7",
     "react-dom": "^15.4.2",
-    "react-dropzone-component": "^1.1.0",
+    "react-dropzone-component": "^1.4.1",
     "react-redux": "^5.0.3",
     "react-router": "^3.0.2",
     "react-router-bootstrap": "^0.23.1",

frontend/webui/src/components/Files.jsx

@@ -16,10 +16,14 @@ class Files extends React.Component {
     constructor(props) {
         super(props);
 
+
+
         this.djsConfig = {
             addRemoveLinks: true,
             autoProcessQueue: false,
             uploadMultiple: true,
+            parallelUploads: 999,
+            maxFilesize: 4000,
             previewTemplate: ReactDOMServer.renderToStaticMarkup(
                 <div className="dz-preview dz-file-preview">
                     <div className="dz-filename"><span data-dz-name="true"></span></div>

frontend/webui/src/components/Jobs.jsx

@@ -30,31 +30,37 @@ class Jobs extends React.Component {
     }
 
     render() {
-        return (
-            <div>
-                <h3>Browse Jobs</h3>
-                <h4>All jobs</h4>
-                <Header/>
-                { this.props.jobs.map((job) => {
-                    let service = null;
-                    for (var i = 0; i < this.props.services.length; ++i) {
-                        if (job.ServiceID == this.props.services[i].ID) {
-                            service = this.props.services[i];
-                            break;
+        if (this.props.jobs) {
+            return (
+                <div>
+                    <h3>Browse Jobs</h3>
+                    <h4>All jobs</h4>
+                    <Header/>
+                    { this.props.jobs.map((job) => {
+                        let service = null;
+                        for (var i = 0; i < this.props.services.length; ++i) {
+                            if (job.ServiceID == this.props.services[i].ID) {
+                                service = this.props.services[i];
+                                break;
+                            }
                         }
-                    }
-                    const serviceName = (service && service.Name && service.Name.length) ? service.Name :
-                        (service && service.ID && service.ID.length) ? service.ID : "unknown service";
-                    const title = "Job from " + serviceName;
+                        const serviceName = (service && service.Name && service.Name.length) ? service.Name :
+                            (service && service.ID && service.ID.length) ? service.ID : "unknown service";
+                        const title = "Job from " + serviceName;
 
-                    if (job.ID === this.props.params.id) {
-                        return <Job key={job.ID} job={job} service={service} title={title}/>
-                    } else {
-                        return <JobRow key={job.ID} job={job} title={title}/>
-                    }
-                })}
-            </div>
-        );
+                        if (job.ID === this.props.params.id) {
+                            return <Job key={job.ID} job={job} service={service} title={title}/>
+                        } else {
+                            return <JobRow key={job.ID} job={job} title={title}/>
+                        }
+                    })}
+                </div>
+            );
+        } else {
+            return (
+                <div><h4>No jobs found</h4></div>
+            )
+        }
     }
 }