Add packer scripts for Drumkit itself, including CLI-only PHP depende…

…ncies script. Also: fix leading @ signs in makefile. We should add a test.
consensus-enterprises · Jul 22, 2021 · 0a8b3d2 · 0a8b3d2
1 parent 0c313fa
commit 0a8b3d2
Show file tree

Hide file tree

Showing 8 changed files with 188 additions and 14 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -10,17 +10,6 @@ variables:
 
 fast-tests: &test-defaults
   stage: test
-  before_script:
-    # Install Git and GNU Make
-    - apt-get update -yqq
-    - apt-get install -yqq git make unzip
-    # Install Behat dependencies
-    - apt-get install -yqq curl php-cli php-mbstring php-curl php-xml php-gd
-    # TODO: Figure out a way to intall these system packages as dependecies of
-    # the targets that use them.
-    - apt-get install -yqq python3-minimal python3-pip python3-yaml python3-jinja2
-    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-    - pip3 install jinja2-cli matrix-client
   script:
     - source d
     - make clean-ansible  # Clean up Ansible config from Drumkit itself.

diff --git a/drumkit/mk.d/20_ci.mk b/drumkit/mk.d/20_ci.mk
@@ -0,0 +1,33 @@
+CONTAINER_REGISTRY_URL ?= https://gitlab.com/consensus.enterprises/drumkit/container_registry
+CONTAINER_PROJECT_NAME = drumkit
+
+local_ref = $(shell git rev-parse HEAD)
+clone_ref = $(shell [ -d .clone ] && (cd .clone && git rev-parse HEAD))
+
+clone: ## Create a local clone of our repo that will be used inside the CI packer image.
+ifneq ($(local_ref),$(clone_ref))
+	@echo "Cloning a fresh copy of our code in to .clone directory."
+	@rm -rf .clone
+	@mkdir -p .clone
+	@git clone --recursive . .clone
+else
+	@echo ".clone directory is up to date, skipping reclone."
+endif
+
+ci-image: packer
+	@if [ -z ${CONTAINER_SCRIPT} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_SCRIPT$(YELLOW).$(RESET)"; exit 1; fi
+	@if [ -z ${CONTAINER_PROJECT_NAME} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_PROJECT_NAME$(YELLOW).$(RESET)"; exit 1; fi
+	@if [ -z ${CONTAINER_REGISTRY_URL} ]; then echo -e "$(YELLOW)Missing required variable $(GREY)CONTAINER_REGISTRY_URL$(YELLOW).$(RESET)"; exit 1; fi
+	@echo "Building packer image for CI: $(CONTAINER_SCRIPT)"
+	@packer build $(CONTAINER_SCRIPT)
+
+ci-images: clone
+	@echo "Building packer images for CI."
+	@echo "Using project name: $(CONTAINER_PROJECT_NAME)"
+	@echo "Using container registry: $(CONTAINER_REGISTRY_URL)"
+	@for script in `ls scripts/packer/json/*.json`; do make -s ci-image CONTAINER_SCRIPT=$$script CONTAINER_PROJECT_NAME=$(CONTAINER_PROJECT_NAME) CONTAINER_REGISTRY_URL=$(CONTAINER_REGISTRY_URL); done
+
+ci-local: gitlab-runner
+	gitlab-runner exec docker tests
+
+
diff --git a/mk/projects/15_packer.mk b/mk/projects/15_packer.mk
@@ -8,9 +8,9 @@ init-project-packer-intro:
 init-project-packer-vars:
 	@echo "Please provide the following information to initialize your Packer scripts:"
 	@read -p "Docker container project name: " project && export CONTAINER_PROJECT_NAME=$${project:-project} && \
-	@read -p "Docker container registry URL: " registry_url && export CONTAINER_REGISTRY_URL=$${registry_url:-registry.gitlab.com} && \
-	@make -s $(MK_D)/20_ci.mk && \
-	@make -s $(PACKER_SH_DIR)/$(CONTAINER_PROJECT_NAME).sh
+	read -p "Docker container registry URL: " registry_url && export CONTAINER_REGISTRY_URL=$${registry_url:-registry.gitlab.com} && \
+	make -s $(MK_D)/20_ci.mk && \
+	make -s $(PACKER_SH_DIR)/$(CONTAINER_PROJECT_NAME).sh
 
 init-project-packer-static: mustache
 	@mkdir -p $(PACKER_JSON_DIR) $(PACKER_SH_DIR)

diff --git a/scripts/packer/json/10-bionic.json b/scripts/packer/json/10-bionic.json
@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "ubuntu:bionic",
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/apt.sh",
+        "scripts/packer/scripts/purge-extra-packages.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "bionic",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+
diff --git a/scripts/packer/json/20-base.json b/scripts/packer/json/20-base.json
@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "{{user `repo_url`}}/bionic:latest",
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/utils.sh",
+        "scripts/packer/scripts/python.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "base",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+
diff --git a/scripts/packer/json/21-php-cli.json b/scripts/packer/json/21-php-cli.json
@@ -0,0 +1,45 @@
+{
+  "builders": [
+    {
+      "commit": true,
+      "image": "{{user `repo_url`}}/base:latest",
+      "pull": false,
+      "type": "docker"
+    }
+  ],
+  "post-processors": [
+    [
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "0.0.x"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "repository": "{{user `repo_url`}}/{{user `image_name`}}",
+        "tags": [
+          "latest"
+        ],
+        "type": "docker-tag"
+      },
+      {
+        "type": "docker-push"
+      }
+    ]
+  ],
+  "provisioners": [
+    {
+      "scripts": [
+        "scripts/packer/scripts/php-cli.sh",
+        "scripts/packer/scripts/cleanup.sh"
+      ],
+      "type": "shell"
+    }
+  ],
+  "variables": {
+    "image_name": "php",
+    "repo_url": "{{env `CONTAINER_REGISTRY_URL`}}"
+  }
+}
+
diff --git a/scripts/packer/scripts/php-cli.sh b/scripts/packer/scripts/php-cli.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -eux
+
+# Use this file for projects that use php on the command line, but don't involve web development;
+# for web development projects, use "php.sh" (the default).
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -yqq update
+# Install Behat dependencies:
+apt-get -yqq install \
+  curl \
+  php-cli \
+  php-mbstring \
+  php-curl \
+  php-xml \
+  php-gd \
+> /dev/null
diff --git a/scripts/packer/scripts/python.sh b/scripts/packer/scripts/python.sh
@@ -12,6 +12,7 @@ apt-get install -yqq \
   python3-apt \
   python3-setuptools \
   python3-wheel \
+  python3-zip \
 > /dev/null
 update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 pip3 install jinja2-cli matrix-client ansible