Merge branch '56-make-packer-images-for-testing' into 'master'

Resolve "Make Packer Images for testing"

Closes #56

See merge request consensus.enterprises/drumkit!47

.gitlab-ci.yml

@@ -1,4 +1,7 @@
-image: ubuntu:bionic
+image: 
+  name: registry.gitlab.com/consensus.enterprises/drumkit/php
+  entrypoint: [""] # We have to override the container entrypoint or else we end up in /bin/sh and `. d` doesn't work. See https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#overriding-the-entrypoint-of-an-image
+
 
 stages:
   - test
@@ -10,17 +13,6 @@ variables:
 
 fast-tests: &test-defaults
   stage: test
-  before_script:
-    # Install Git and GNU Make
-    - apt-get update -yqq
-    - apt-get install -yqq git make unzip
-    # Install Behat dependencies
-    - apt-get install -yqq curl php-cli php-mbstring php-curl php-xml php-gd
-    # TODO: Figure out a way to intall these system packages as dependecies of
-    # the targets that use them.
-    - apt-get install -yqq python3-minimal python3-pip python3-yaml python3-jinja2
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-    - pip3 install jinja2-cli matrix-client
   script:
     - source d
     - make clean-ansible  # Clean up Ansible config from Drumkit itself.

drumkit/mk.d/20_ci.mk

@@ -0,0 +1,31 @@
+CONTAINER_REGISTRY_URL ?= registry.gitlab.com/consensus.enterprises/drumkit
+CONTAINER_PROJECT_NAME = drumkit
+
+local_ref = $(shell git rev-parse HEAD)
+clone_ref = $(shell [ -d .clone ] && (cd .clone && git rev-parse HEAD))
+
+clone: ## Create a local clone of our repo that will be used inside the CI packer image.
+ifneq ($(local_ref),$(clone_ref))
+	@echo "Cloning a fresh copy of our code in to .clone directory."
+	@rm -rf .clone
+	@mkdir -p .clone
+	@git clone --recursive . .clone
+else
+	@echo ".clone directory is up to date, skipping reclone."
+endif
+
+ci-image: packer
+	@if [ -z ${CONTAINER_SCRIPT} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_SCRIPT$(YELLOW).$(RESET)"; exit 1; fi
+	@if [ -z ${CONTAINER_PROJECT_NAME} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_PROJECT_NAME$(YELLOW).$(RESET)"; exit 1; fi
+	@if [ -z ${CONTAINER_REGISTRY_URL} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_REGISTRY_URL$(YELLOW).$(RESET)"; exit 1; fi
+	@echo "Building packer image for CI: $(CONTAINER_SCRIPT)"
+	@packer build $(CONTAINER_SCRIPT)
+
+ci-images: clone
+	@echo "Building packer images for CI."
+	@echo "Using project name: $(CONTAINER_PROJECT_NAME)"
+	@echo "Using container registry: $(CONTAINER_REGISTRY_URL)"
+	@for script in `ls scripts/packer/json/*.json`; do make -s ci-image CONTAINER_SCRIPT=$$script CONTAINER_PROJECT_NAME=$(CONTAINER_PROJECT_NAME) CONTAINER_REGISTRY_URL=$(CONTAINER_REGISTRY_URL); done
+
+ci-local: gitlab-runner
+	gitlab-runner exec docker tests

mk/projects/15_packer.mk

@@ -8,9 +8,9 @@ init-project-packer-intro:
 init-project-packer-vars:
 	@echo "Please provide the following information to initialize your Packer scripts:"
 	@read -p "Docker container project name: " project && export CONTAINER_PROJECT_NAME=$${project:-project} && \
-	@read -p "Docker container registry URL: " registry_url && export CONTAINER_REGISTRY_URL=$${registry_url:-registry.gitlab.com} && \
-	@make -s $(MK_D)/20_ci.mk && \
-	@make -s $(PACKER_SH_DIR)/$(CONTAINER_PROJECT_NAME).sh
+	read -p "Docker container registry URL: " registry_url && export CONTAINER_REGISTRY_URL=$${registry_url:-registry.gitlab.com} && \
+	make -s $(MK_D)/20_ci.mk && \
+	make -s $(PACKER_SH_DIR)/$(CONTAINER_PROJECT_NAME).sh
 
 init-project-packer-static: mustache
 	@mkdir -p $(PACKER_JSON_DIR) $(PACKER_SH_DIR)

scripts/packer/json/10-bionic.json

@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "ubuntu:bionic",
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/apt.sh",
+        "scripts/packer/scripts/purge-extra-packages.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "bionic",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+

scripts/packer/json/20-base.json

@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "{{user `repo_url`}}/bionic:latest",
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/utils.sh",
+        "scripts/packer/scripts/python.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "base",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+

scripts/packer/json/21-php-cli.json

@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "{{user `repo_url`}}/base:latest",
+      "pull": false,
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/php-cli.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "php",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+

scripts/packer/scripts/php-cli.sh

@@ -0,0 +1,16 @@
+#!/bin/sh -eux
+
+# Use this file for projects that use php on the command line, but don't involve web development;
+# for web development projects, use "php.sh" (the default).
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -yqq update
+# Install Behat dependencies:
+apt-get -yqq install \
+  curl \
+  php-cli \
+  php-mbstring \
+  php-curl \
+  php-xml \
+  php-gd \
+> /dev/null

scripts/packer/scripts/python.sh

@@ -13,5 +13,6 @@ apt-get install -yqq \
   python3-setuptools \
   python3-wheel \
 > /dev/null
+pip3 install --upgrade pip
 update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 pip3 install jinja2-cli matrix-client ansible