kuttl integration test

Run an e2e test with ephemeral kind cluster.
Usage: REGISTRY=<docker-registry> make e2e-test

Signed-off-by: Leonardo Milleri <[email protected]>

Makefile

@@ -172,6 +172,15 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default > dist/install.yaml
 
+# Run sample attestation in a kind cluster
+# pre-requirements: k8s kuttl plugin installed
+# you can change the trustee image by defining the env variable KBS_IMAGE_NAME
+# Usage: REGISTRY=<user-registry> KBS_IMAGE_NAME=<trustee-image> make test-e2e
+.PHONY: test-e2e
+test-e2e:
+	@[ "${REGISTRY}" ] || ( echo ">> REGISTRY is not set. Usage: REGISTRY=<user-registry> make test-e2e"; exit 1 )
+	kubectl kuttl test
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to

README.md

@@ -312,6 +312,18 @@ which provide a reconcile function responsible for synchronizing resources until
 
 **NOTE:** You can also run this in one step by running: `make install run`
 
+### Integration tests
+
+An attestation with the sample-attester is performed in an ephemeral kind cluster
+Pre-requirements:
+- k8s [kuttl](https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin) plugin installed
+
+Optional: set the env variable KBS_IMAGE_NAME to override the default trustee image
+
+  ```sh
+  REGISTRY=<docker-registry> make test-e2e
+  ```
+
 ### Modifying the API definitions
 
 If you are editing the API definitions, generate the manifests such as CRs or CRDs using:

kuttl-test.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+startKIND: true
+testDirs:
+- tests/e2e/
+timeout: 120

tests/e2e/sample-attester/00-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trustee-operator-controller-manager
+  namespace: trustee-operator-system
+status:
+  readyReplicas: 1

tests/e2e/sample-attester/00-install.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./install-operator.sh

tests/e2e/sample-attester/01-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kbs-auth-public-key
+  namespace: trustee-operator-system

tests/e2e/sample-attester/01-auth-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./create-auth-secret.sh

tests/e2e/sample-attester/02-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kbsres1
+  namespace: trustee-operator-system

tests/e2e/sample-attester/02-kbs-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./create-other-secret.sh

tests/e2e/sample-attester/03-kbs-config.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kbs-config
+  namespace: trustee-operator-system
+data:
+  kbs-config.json: |
+    {
+        "insecure_http" : true,
+        "sockets": ["0.0.0.0:8080"],
+        "auth_public_key": "/etc/auth-secret/publicKey",
+        "attestation_token_config": {
+          "attestation_token_type": "CoCo"
+        },
+        "repository_config": {
+          "type": "LocalFs",
+          "dir_path": "/opt/confidential-containers/kbs/repository"
+        },
+        "as_config": {
+          "work_dir": "/opt/confidential-containers/attestation-service",
+          "policy_engine": "opa",
+          "attestation_token_broker": "Simple",
+          "attestation_token_config": {
+            "duration_min": 5
+          },
+          "rvps_config": {
+            "store_type": "LocalJson",
+            "store_config": {
+              "file_path": "/opt/confidential-containers/rvps/reference-values/reference-values.json"
+            }
+          }
+        },
+        "policy_engine_config": {
+          "policy_path": "/opt/confidential-containers/opa/policy.rego"
+        }
+    }

tests/e2e/sample-attester/04-resource-policy.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: resource-policy
+  namespace: trustee-operator-system
+data:
+  policy.rego: |
+    package policy
+
+    default allow = false
+    allow {
+            input["tcb-status"]["sample.svn"] == "1"
+    }

tests/e2e/sample-attester/05-rvps-reference-values.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rvps-reference-values
+  namespace: trustee-operator-system
+data:
+  reference-values.json: |
+    [
+    ]

tests/e2e/sample-attester/06-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trustee-deployment
+  namespace: trustee-operator-system
+status:
+  readyReplicas: 1

tests/e2e/sample-attester/06-kbsconfig_sample.yaml

@@ -0,0 +1,21 @@
+apiVersion: confidentialcontainers.org/v1alpha1
+kind: KbsConfig
+metadata:
+  labels:
+    app.kubernetes.io/name: kbsconfig
+    app.kubernetes.io/instance: kbsconfig-sample
+    app.kubernetes.io/part-of: trustee-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: trustee-operator
+  name: kbsconfig-sample
+  namespace: trustee-operator-system
+spec:
+  kbsConfigMapName: kbs-config
+  kbsAuthSecretName: kbs-auth-public-key
+  kbsDeploymentType: AllInOneDeployment
+  kbsRvpsRefValuesConfigMapName: rvps-reference-values
+  kbsResourcePolicyConfigMapName: resource-policy
+  kbsSecretResources:
+    - "kbsres1"
+  KbsEnvVars: 
+    RUST_LOG: debug  

tests/e2e/sample-attester/07-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kbs-client
+  namespace: trustee-operator-system
+status:
+  containerStatuses:
+  - ready: true

tests/e2e/sample-attester/07-client.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kbs-client
+  namespace: trustee-operator-system
+spec:
+  containers:
+  - name: kbs-client
+    image: quay.io/confidential-containers/kbs-client:v0.9.0
+    imagePullPolicy: IfNotPresent
+    command:
+      - sleep
+      - "360000"
+    env:
+      - name: RUST_LOG
+        value:  none

tests/e2e/sample-attester/08-assert.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trustee-secret
+  namespace: trustee-operator-system

tests/e2e/sample-attester/08-get-secret.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: ./get-secret.sh

tests/e2e/sample-attester/create-auth-secret.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+openssl genpkey -algorithm ed25519 > privateKey
+openssl pkey -in privateKey -pubout -out publicKey
+kubectl create secret generic kbs-auth-public-key --from-file=publicKey -n trustee-operator-system

tests/e2e/sample-attester/create-other-secret.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubectl create secret generic kbsres1 --from-literal key1=res1val1 --from-literal key2=res1val2 -n trustee-operator-system

tests/e2e/sample-attester/get-secret.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+SECRET="$(kubectl exec -it -n trustee-operator-system kbs-client -- kbs-client --url http://kbs-service:8080 get-resource --path default/kbsres1/key1)"
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error when retrieving the secret"
+else
+    echo "Attestation completed successfully: secret="$SECRET
+    # this secret is created only to check the secret retrieval has been successful
+    kubectl create secret generic trustee-secret --from-literal key1=$SECRET -n trustee-operator-system
+fi

tests/e2e/sample-attester/install-operator.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+KBS_IMAGE_NAME="${KBS_IMAGE_NAME:-quay.io/confidential-containers/trustee:290fd0eb64ab20f50efbd27cf80542851c0ee17f}"
+export IMG=${REGISTRY}/trustee-operator:test
+
+pushd ../../..
+
+pushd config/manager
+kustomize edit set image controller=$IMG
+kustomize edit add patch --patch "- op: replace
+  path: '/spec/template/spec/containers/0/env/1'
+  value:
+    name: KBS_IMAGE_NAME
+    value: ${KBS_IMAGE_NAME}" --kind Deployment --name controller-manager
+popd
+make docker-build docker-push
+make build-installer
+kubectl apply -f dist/install.yaml
+
+popd