Refactoring, adapters separation. Custom resource kinds support. (#93)

* Refactoring, adapters separation. Custom resource adapter and support. Added cr-related info to Readme.

Evaluate plural name of CRD via API call and qualifier filtration. Basic tests.

* Get rid of Qualifier scheme. Evaluate plural via custom extended ResourceAPI request.

* Support default namespace in custom kinds adapter

README.md

@@ -586,4 +586,13 @@ $ k8s-handle delete -r service.yaml --use-kubeconfig
 2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Using namespace "test"
 2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Trying to delete Service "k8s-handle-example"
 2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Service "k8s-handle-example" deleted
-```
+```
+
+### Custom resource definitions and custom resources
+Since version 0.5.5 k8s-handle supports Custom resource definition (CRD) and custom resource (CR) kinds.
+If your deployment involves use of such kinds, make sure that CRD was deployed before CR and check correctness of the CRD's scope.
+
+Currently, in the case of a custom resource deployment, k8s-handle takes into account only the namespace 
+specified in the `metadata.namespace` field of the spec, in order to distinguish whether CR should be deployed/removed via namespaced API or not, without additional markers or flags. 
+
+Default k8s_namespace (taken from CLI, env or context) **is not applied** to such resources. Therefore, in order to create namespaced CR, one should explicitly set `metadata.namespace` field of that resource's spec.

k8s_handle/__init__.py

@@ -11,9 +11,10 @@
 from k8s_handle import config
 from k8s_handle import settings
 from k8s_handle import templating
+from k8s_handle.exceptions import DeprecationError, ProvisioningError
 from k8s_handle.filesystem import InvalidYamlError
-from k8s_handle.k8s.deprecation_checker import ApiDeprecationChecker, DeprecationError
-from k8s_handle.k8s.resource import Provisioner, ProvisioningError
+from k8s_handle.k8s.deprecation_checker import ApiDeprecationChecker
+from k8s_handle.k8s.provisioner import Provisioner
 
 COMMAND_DEPLOY = 'deploy'
 COMMAND_DESTROY = 'destroy'

k8s_handle/exceptions.py

@@ -0,0 +1,14 @@
+class ProvisioningError(Exception):
+    pass
+
+
+class DeprecationError(Exception):
+    pass
+
+
+class InvalidYamlError(Exception):
+    pass
+
+
+class TemplateRenderingError(Exception):
+    pass

k8s_handle/filesystem.py

@@ -5,15 +5,13 @@
 
 import yaml
 
+from k8s_handle.exceptions import InvalidYamlError
+
 # furiousassault RE: it's not a good practice to log from utility function
 # maybe we should pass os.remove failure silently, it doesn't seem so important
 log = logging.getLogger(__name__)
 
 
-class InvalidYamlError(Exception):
-    pass
-
-
 def load_yaml(path):
     try:
         with open(path) as f:

k8s_handle/k8s/adapters.py

@@ -0,0 +1,295 @@
+import logging
+
+from kubernetes import client
+from kubernetes.client.rest import ApiException
+
+from k8s_handle import settings
+from k8s_handle.exceptions import ProvisioningError
+from k8s_handle.transforms import add_indent, split_str_by_capital_letters
+from .api_extensions import ResourcesAPI
+from .mocks import K8sClientMock
+
+log = logging.getLogger(__name__)
+
+
+class Adapter:
+    api_versions = {
+        'apps/v1beta1': client.AppsV1beta1Api,
+        'v1': client.CoreV1Api,
+        'extensions/v1beta1': client.ExtensionsV1beta1Api,
+        'batch/v1': client.BatchV1Api,
+        'batch/v2alpha1': client.BatchV2alpha1Api,
+        'batch/v1beta1': client.BatchV1beta1Api,
+        'policy/v1beta1': client.PolicyV1beta1Api,
+        'storage.k8s.io/v1': client.StorageV1Api,
+        'apps/v1': client.AppsV1Api,
+        'autoscaling/v1': client.AutoscalingV1Api,
+        'rbac.authorization.k8s.io/v1': client.RbacAuthorizationV1Api,
+        'scheduling.k8s.io/v1alpha1': client.SchedulingV1alpha1Api,
+        'scheduling.k8s.io/v1beta1': client.SchedulingV1beta1Api,
+        'networking.k8s.io/v1': client.NetworkingV1Api,
+        'apiextensions.k8s.io/v1beta1': client.ApiextensionsV1beta1Api,
+    }
+    kinds_builtin = [
+        'ConfigMap', 'CronJob', 'DaemonSet', 'Deployment', 'Endpoints',
+        'Ingress', 'Job', 'Namespace', 'PodDisruptionBudget', 'ResourceQuota',
+        'Secret', 'Service', 'ServiceAccount', 'StatefulSet', 'StorageClass',
+        'PersistentVolume', 'PersistentVolumeClaim', 'HorizontalPodAutoscaler',
+        'Role', 'RoleBinding', 'ClusterRole', 'ClusterRoleBinding', 'CustomResourceDefinition',
+        'PriorityClass', 'PodSecurityPolicy', 'LimitRange', 'NetworkPolicy'
+    ]
+
+    def __init__(self, spec):
+        self.body = spec
+        self.kind = spec.get('kind', "")
+        self.name = spec.get('metadata', {}).get('name')
+        self.namespace = spec.get('metadata', {}).get('namespace', "") or settings.K8S_NAMESPACE
+
+    @staticmethod
+    def get_instance(spec, api_custom_objects=None, api_resources=None):
+        # due to https://github.com/kubernetes-client/python/issues/387
+        if spec.get('kind') in Adapter.kinds_builtin:
+            if spec.get('apiVersion') == 'test/test':
+                return AdapterBuiltinKind(spec, K8sClientMock(spec.get('metadata', {}).get('name')))
+
+            api = Adapter.api_versions.get(spec.get('apiVersion'))
+
+            if not api:
+                return None
+
+            return AdapterBuiltinKind(spec, api())
+
+        api_custom_objects = api_custom_objects or client.CustomObjectsApi()
+        api_resources = api_resources or ResourcesAPI()
+        return AdapterCustomKind(spec, api_custom_objects, api_resources)
+
+
+class AdapterBuiltinKind(Adapter):
+    def __init__(self, spec, api=None):
+        super().__init__(spec)
+        self.kind = split_str_by_capital_letters(spec['kind'])
+        self.replicas = spec.get('spec', {}).get('replicas')
+        self.api = api
+
+    def get(self):
+        try:
+            if hasattr(self.api, "read_namespaced_{}".format(self.kind)):
+                response = getattr(self.api, 'read_namespaced_{}'.format(self.kind))(
+                    self.name, namespace=self.namespace)
+            else:
+                response = getattr(self.api, 'read_{}'.format(self.kind))(self.name)
+        except ApiException as e:
+            if e.reason == 'Not Found':
+                return None
+            log.error('Exception when calling "read_namespaced_{}": {}'.format(self.kind, add_indent(e.body)))
+            raise ProvisioningError(e)
+
+        return response
+
+    def get_pods_by_selector(self, label_selector):
+        try:
+            if not isinstance(self.api, K8sClientMock):
+                self.api = client.CoreV1Api()
+
+            return self.api.list_namespaced_pod(
+                namespace=self.namespace, label_selector='job-name={}'.format(label_selector))
+
+        except ApiException as e:
+            log.error('Exception when calling CoreV1Api->list_namespaced_pod: {}', e)
+            raise e
+
+    def read_pod_status(self, name):
+        try:
+            if not isinstance(self.api, K8sClientMock):
+                self.api = client.CoreV1Api()
+
+            return self.api.read_namespaced_pod_status(name, namespace=self.namespace)
+        except ApiException as e:
+            log.error('Exception when calling CoreV1Api->read_namespaced_pod_status: {}', e)
+            raise e
+
+    def read_pod_logs(self, name, container):
+        log.info('Read logs for pod "{}", container "{}"'.format(name, container))
+        try:
+            if not isinstance(self.api, K8sClientMock):
+                self.api = client.CoreV1Api()
+            if settings.COUNT_LOG_LINES:
+                return self.api.read_namespaced_pod_log(name, namespace=self.namespace, timestamps=True,
+                                                        tail_lines=settings.COUNT_LOG_LINES, container=container)
+            return self.api.read_namespaced_pod_log(name, namespace=self.namespace, timestamps=True,
+                                                    container=container)
+        except ApiException as e:
+            log.error('Exception when calling CoreV1Api->read_namespaced_pod_log: {}', e)
+            raise e
+
+    def create(self):
+        try:
+            if hasattr(self.api, "create_namespaced_{}".format(self.kind)):
+                return getattr(self.api, 'create_namespaced_{}'.format(self.kind))(
+                    body=self.body, namespace=self.namespace)
+
+            return getattr(self.api, 'create_{}'.format(self.kind))(body=self.body)
+        except ApiException as e:
+            log.error('Exception when calling "create_namespaced_{}": {}'.format(self.kind, add_indent(e.body)))
+            raise ProvisioningError(e)
+        except ValueError as e:
+            log.error(e)
+            # WORKAROUND https://github.com/kubernetes-client/python/issues/466
+            # also https://github.com/kubernetes-client/gen/issues/52
+            if self.kind not in ['pod_disruption_budget', 'custom_resource_definition']:
+                raise e
+
+    def replace(self, parameters):
+        try:
+            if self.kind in ['custom_resource_definition']:
+                self.body['metadata']['resourceVersion'] = parameters['resourceVersion']
+                return self.api.replace_custom_resource_definition(
+                    self.name, self.body,
+                )
+
+            if self.kind in ['service', 'service_account']:
+                if 'spec' in self.body:
+                    self.body['spec']['ports'] = parameters.get('ports')
+
+                return getattr(self.api, 'patch_namespaced_{}'.format(self.kind))(
+                    name=self.name, body=self.body, namespace=self.namespace
+                )
+
+            if hasattr(self.api, "replace_namespaced_{}".format(self.kind)):
+                return getattr(self.api, 'replace_namespaced_{}'.format(self.kind))(
+                    name=self.name, body=self.body, namespace=self.namespace)
+
+            return getattr(self.api, 'replace_{}'.format(self.kind))(
+                name=self.name, body=self.body)
+        except ApiException as e:
+            log.error('Exception when calling "replace_namespaced_{}": {}'.format(self.kind, add_indent(e.body)))
+            raise ProvisioningError(e)
+
+    def delete(self):
+        try:
+            if hasattr(self.api, "delete_namespaced_{}".format(self.kind)):
+                return getattr(self.api, 'delete_namespaced_{}'.format(self.kind))(
+                    name=self.name, body=client.V1DeleteOptions(propagation_policy='Foreground'),
+                    namespace=self.namespace)
+
+            return getattr(self.api, 'delete_{}'.format(self.kind))(
+                name=self.name, body=client.V1DeleteOptions(propagation_policy='Foreground'))
+        except ApiException as e:
+            if e.reason == 'Not Found':
+                return None
+            log.error('Exception when calling "delete_namespaced_{}": {}'.format(self.kind, add_indent(e.body)))
+            raise ProvisioningError(e)
+
+
+class AdapterCustomKind(Adapter):
+    def __init__(self, spec, api_custom_objects, api_resources):
+        super().__init__(spec)
+        self.api = api_custom_objects
+        self.api_resources = api_resources
+        self.plural = None
+
+        try:
+            api_version_splitted = spec.get('apiVersion').split('/', 1)
+            self.group = api_version_splitted[0]
+            self.version = api_version_splitted[1]
+        except (IndexError, AttributeError):
+            self.group = None
+            self.version = None
+
+        resources_list = self.api_resources.list_api_resource_arbitrary(self.group, self.version)
+
+        if not resources_list:
+            return
+
+        for resource in resources_list.resources:
+            if resource.kind != self.kind:
+                continue
+
+            self.plural = resource.name
+
+            if not resource.namespaced:
+                self.namespace = ""
+
+            break
+
+    def get(self):
+        self._validate()
+
+        try:
+            if self.namespace:
+                return self.api.get_namespaced_custom_object(
+                    self.group, self.version, self.namespace, self.plural, self.name
+                )
+
+            return self.api.get_cluster_custom_object(self.group, self.version, self.plural, self.name)
+
+        except ApiException as e:
+            if e.reason == 'Not Found':
+                return None
+
+            log.error('{}'.format(add_indent(e.body)))
+            raise ProvisioningError(e)
+
+    def create(self):
+        self._validate()
+
+        try:
+            if self.namespace:
+                return self.api.create_namespaced_custom_object(
+                    self.group, self.version, self.namespace, self.plural, self.body
+                )
+
+            return self.api.create_cluster_custom_object(self.group, self.version, self.plural, self.body)
+
+        except ApiException as e:
+            log.error('{}'.format(add_indent(e.body)))
+            raise ProvisioningError(e)
+
+    def delete(self):
+        self._validate()
+
+        try:
+            if self.namespace:
+                return self.api.delete_namespaced_custom_object(
+                    self.group, self.version, self.namespace, self.plural, self.name,
+                    client.V1DeleteOptions(propagation_policy='Foreground')
+                )
+
+            return self.api.delete_cluster_custom_object(
+                self.group, self.version, self.plural, self.name,
+                client.V1DeleteOptions(propagation_policy='Foreground')
+            )
+
+        except ApiException as e:
+            if e.reason == 'Not Found':
+                return None
+
+            log.error(
+                '{}'.format(add_indent(e.body)))
+            raise ProvisioningError(e)
+
+    def replace(self, _):
+        self._validate()
+
+        try:
+            if self.namespace:
+                return self.api.patch_namespaced_custom_object(
+                    self.group, self.version, self.namespace, self.plural, self.name, self.body
+                )
+
+            return self.api.patch_cluster_custom_object(
+                self.group, self.version, self.plural, self.name, self.body
+            )
+        except ApiException as e:
+            log.error('{}'.format(add_indent(e.body)))
+            raise ProvisioningError(e)
+
+    def _validate(self):
+        if not self.plural:
+            raise RuntimeError("No valid plural name of resource definition discovered")
+
+        if not self.group:
+            raise RuntimeError("No valid resource definition group discovered")
+
+        if not self.version:
+            raise RuntimeError("No valid version of resource definition supplied")