changes for uploading disc_format

add qemu-utils
introducing a number of fixes and shellcheck
specify upload image_size
minor fix

Dockerfile

@@ -21,6 +21,8 @@ ENV PLUGIN_AZURERM=0.2.2
 ENV PLUGIN_NULL=1.0.0
 ENV PLUGIN_CLOUDFLARE=0.1.0
 ENV PLUGIN_TEMPLATE=1.0.0
+# Pip version
+ENV PIP=9.0.3
 
 # Install with apt and pip
 RUN apt-get update -y && apt-get install -y \
@@ -33,6 +35,7 @@ RUN apt-get update -y && apt-get install -y \
       libffi-dev \
       openssl \
       python-pip \
+      qemu-utils \
       unzip && \
     `# Add google cloud` \
     echo "deb http://packages.cloud.google.com/apt cloud-sdk-xenial main" \
@@ -41,9 +44,8 @@ RUN apt-get update -y && apt-get install -y \
       | apt-key add - && \
     apt-get update -y && apt-get install -y \
       google-cloud-sdk="$GOOGLE_CLOUD_SDK_VERSION" && \
-    `# Pip` \
-    pip install --no-cache-dir --upgrade \
-      pip && \
+    `# Upgrade pip and install pip deps` \
+    pip install --no-cache-dir --upgrade pip=="${PIP}" && \
     pip install --no-cache-dir \
       ansible=="$ANSIBLE_VERSION" \
       j2cli=="$J2CLI_VERSION" \

bin/image-create-openstack.sh

@@ -1,101 +1,40 @@
 #!/usr/bin/env bash
 
 # Checks and uploads specified Image to user's OpenStack account.
-# Downloads image first from an Amazon S3 storage account to host,
-# then uploads image to openstack teenancy.
-# Uses curl and python-glanceclient to do the job.
+#
+# 1. Downloads image first from an Amazon S3 storage (using curl and python-glance client)
+#
+# 2. Converts the image to the required format (if needed)
+#
+# 3. Uploads image to openstack tenancy. (using curl and python-glance client)
+#
+#
 #
 # Env vars
+#
 #   KN_IMAGE_NAME
+#   KN_DISC_FORMAT
 #   KN_IMAGE_BUCKET_URL
-
+#
 # Exit immediately if a command exits with a non-zero status
+
 set -e
 
+source ./bin/openstack/functions.sh
+KN_IMAGE_BUCKET_URL=${KN_IMAGE_BUCKET_URL:-"https://s3.eu-central-1.amazonaws.com/kubenow-eu-central-1"}
+
 echo "Started script image-create-openstack"
 
-if [ -z "$KN_IMAGE_NAME" ]; then
+if [ -z "${KN_IMAGE_NAME}" ]; then
   echo >&2 "env KN_IMAGE_NAME must be set for this script to run"
   exit 1
 fi
 
-KN_IMAGE_BUCKET_URL=${KN_IMAGE_BUCKET_URL:-"https://s3.eu-central-1.amazonaws.com/kubenow-eu-central-1"}
-file_name="$KN_IMAGE_NAME.qcow2"
-
-# Check if image is present already
-echo "List images available in OpenStack..."
-image_list="$(glance image-list)"
-image_id="$(printf '%s' "$image_list" |
-  grep "\s$KN_IMAGE_NAME\s" |
-  awk -F "|" '{print $2;}' |
-  tr -d '[:space:]')"
-
-# If it doesn't exist then download it
-if [ -z "$image_id" ]; then
-  echo "Image not present in OpenStack"
-  echo "Downloading image to local /tmp/"
-  curl "$KN_IMAGE_BUCKET_URL/$file_name" \
-    -o "/tmp/$file_name" \
-    --connect-timeout 30 \
-    --max-time 1800
-
-  echo "Download md5 sum file"
-  curl "$KN_IMAGE_BUCKET_URL/$file_name.md5" \
-    -o "/tmp/$file_name.md5" \
-    --connect-timeout 30 \
-    --max-time 1800
-
-  echo "Check md5 sum"
-  md5only=$(cut -f1 -d ' ' "/tmp/$file_name.md5")
-  printf '%s' "$md5only  /tmp/$file_name" | md5sum -c
-else
-  echo "File exists - no need to download"
-fi
-
-# If it didn't exist then upload it
-if [ -z "$image_id" ]; then
-  echo "Uploading image"
-  glance image-create \
-    --file "/tmp/$file_name" \
-    --disk-format qcow2 \
-    --min-disk 20 \
-    --container-format bare \
-    --name "$KN_IMAGE_NAME" \
-    --progress
-else
-  echo "file exists - no need to upload"
-fi
-
-echo "Verify md5 of present/uploaded image..."
-echo "List images available in OpenStack..."
-image_list="$(glance image-list)"
-image_id="$(printf '%s' "$image_list" |
-  grep "\s$KN_IMAGE_NAME\s" |
-  awk -F "|" '{print $2;}' |
-  tr -d '[:space:]')"
-
-# Get checksum of uploaded file
-image_details="$(glance image-show "$image_id")"
-checksum="$(printf '%s' "$image_details" |
-  grep -w "checksum" |
-  awk -F "|" '{print $3;}' |
-  tr -d '[:space:]')"
-
-# Get checksum of bucket image
-echo "Download md5 sum file"
-curl "$KN_IMAGE_BUCKET_URL/$file_name.md5" \
-  -o "/tmp/$file_name.md5" \
-  --connect-timeout 30 \
-  --max-time 1800
-
-md5only=$(cut -f1 -d ' ' "/tmp/$file_name.md5")
-if [ "$md5only" != "$checksum" ]; then
-  echo >&2 "Wrong checksum of present/uploaded image."
-  echo >&2 "Something might have failed on file transfer."
-  echo >&2 "Please delete image $KN_IMAGE_NAME from Openstack and try again."
+if [ -z "${KN_DISC_FORMAT}" ]; then
+  echo >&2 "env KN_DISC_FORMAT must be set for this script to run"
   exit 1
-else
-  echo "Checksum OK"
 fi
 
-echo "Image upload done"
+maybe_download_image "${KN_IMAGE_NAME}" "${KN_DISC_FORMAT}" "${KN_IMAGE_BUCKET_URL}"
+maybe_convert_image "${KN_IMAGE_NAME}" "${KN_DISC_FORMAT}"
+maybe_upload_image "${KN_IMAGE_NAME}" "${KN_DISC_FORMAT}" "${KN_IMAGE_BUCKET_URL}"

bin/kn-apply

@@ -11,6 +11,10 @@ image_name=$(grep -w "boot_image" terraform.tfvars |
   cut -d "=" -f 2- |
   awk -F\" '{print $(NF-1)}')
 
+disc_format=$(grep -w "disc_format" terraform.tfvars |
+  cut -d "=" -f 2- |
+  awk -F\" '{print $(NF-1)}')
+
 # Check for recognized cloud provider
 if ! grep -qw "$host_cloud" <<<"openstack gce azure aws"; then
   echo >&2 "Error: unrecognized host cloud '$host_cloud' in config file terraform.tfvars"
@@ -22,6 +26,7 @@ if [ "$host_cloud" = 'openstack' ] || [ "$host_cloud" = 'gce' ] || [ "$host_clou
   export KN_GCE_ACCOUNT_FILE_PATH="$PWD/service-account.json"
   export TF_VARS_FILE="$PWD/terraform.tfvars"
   export KN_IMAGE_NAME="$image_name"
+  export KN_DISC_FORMAT="$disc_format"
   /KubeNow_root/bin/image-create-"$host_cloud".sh
 fi