From e9b185b26482fb18ad32cb4772270900bc3a37e9 Mon Sep 17 00:00:00 2001 From: Ilya Golovchenko <37325304+furiousassault@users.noreply.github.com> Date: Mon, 18 Feb 2019 14:32:47 +0700 Subject: [PATCH] Add support of render, apply, delete commands, entrypoint refactoring. (#87) * Add support of render, apply, delete commands, entrypoint refactoring, source cleanup, changes in README --- README.md | 116 +++++++++++++++------- k8s_handle/__init__.py | 208 +++++++++++++++++++++++++++------------ k8s_handle/settings.py | 3 - k8s_handle/templating.py | 149 ++++++++++++++-------------- tests/test_templating.py | 18 ++-- 5 files changed, 308 insertions(+), 186 deletions(-) diff --git a/README.md b/README.md index e539a83..56b01cf 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ k8s-handle is a helm alternative, but without package manager * [Before you begin](#before-you-begin) * [Installation with pip](#installation-with-pip) * [Usage with docker](#usage-with-docker) -* [Using with CI/CD tools](#using-with-cicd-tools) +* [Usage with CI/CD tools](#usage-with-cicd-tools) * [Usage](#usage) * [Example](#example) * [Docs](#docs) @@ -31,10 +31,13 @@ k8s-handle is a helm alternative, but without package manager * [Native integration](#native-integration) * [Through variables](#through-variables) * [Working modes](#working-modes) - * [Dry run](#dry-run) * [Sync mode](#sync-mode) * [Strict mode](#strict-mode) * [Destroy](#destroy) + * [Operating without config.yaml](#operating-without-configyaml) + * [Render](#render) + * [Apply](#apply) + * [Delete](#delete) # Features * Easy to use command line interface @@ -118,7 +121,7 @@ INFO:k8s.resource:Deployment "k8s-starter-kit" does not exist, create it ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``` -# Using with CI/CD tools +# Usage with CI/CD tools If you using Gitlab CI, TeamCity or something else, you can use docker runner/agent, script will slightly different: ```bash $ k8s-handle deploy -s staging @@ -214,8 +217,8 @@ Deployed with k8s-handle. # Docs ## Configuration structure -k8s-handle work with 2 components: - * conifg.yaml (or any other yaml file through -c argument) store all configuration for deploy +k8s-handle works with 2 components: + * config.yaml (or any other yaml file through -c argument) that stores all configuration for deploy * templates catalog, where your can store all required templates for kubernetes resource files (can be changed through TEMPLATES_DIR env var) @@ -366,7 +369,7 @@ config.yaml, but you can do it in another way if necessary. Thus, the k8s-handle provides flexible ways to set the required parameters. ### Merging with common -All variables defined in common merged with deployed section and available as context dict in templates rendering, +All variables defined in `common` will be merged with deployed section and available as context dict in templates rendering, for example: ```yaml common: @@ -374,12 +377,12 @@ common: testing: testing_variable: testing_value ``` -After rendering this template some-file.txt.j2: +After the rendering of this template some-file.txt.j2: ```txt common_var = {{ common_var }} testing_variable = {{ testing_variable }} ``` -will be generated file some-file.txt with content: +file some-file.txt will be generated with the following content: ```txt common_var = common_value testing_variable = testing_value @@ -458,30 +461,6 @@ deploy: - k8s-handle deploy --section ``` ## Working modes -### Dry run -If you want check templates generation and not apply changes to kubernetes use --dry-run function. -```bash -$ k8s-handle deploy -s staging --use-kubeconfig --dry-run -INFO:templating:Trying to generate file from template "configmap.yaml.j2" in "/tmp/k8s-handle" -INFO:templating:File "/tmp/k8s-handle/configmap.yaml" successfully generated -INFO:templating:Trying to generate file from template "deployment.yaml.j2" in "/tmp/k8s-handle" -INFO:templating:File "/tmp/k8s-handle/deployment.yaml" successfully generated -INFO:templating:Trying to generate file from template "service.yaml.j2" in "/tmp/k8s-handle" -INFO:templating:File "/tmp/k8s-handle/service.yaml" successfully generated -$ cat /tmp/k8s-handle/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: example -spec: - type: NodePort - ports: - - name: http - port: 80 - targetPort: 80 - selector: - app: example -``` ### Sync mode > Works only with Deployment, Job, StatefulSet and DaemonSet @@ -513,8 +492,77 @@ ERROR:__main__:RuntimeError: Environment variable "IMAGE_VERSION" is not set $ echo $? 1 ``` -## Destroy +### Destroy In some cases you need to destroy early created resources(demo env, deploy from git branches, testing etc.), k8s-handle support `destroy` subcommand for you. Just use `destroy` instead of `deploy`. k8s-handle process destroy as deploy, but call delete kubernetes api calls instead of create or replace. -> Sync mode available for destroy too +> Sync mode is available for destroy as well. + +## Operating without config.yaml +The most common way for the most of use cases is to operate with k8s-handle via `config.yaml`, specifying +connection parameters, targets (sections and tags) and variables in one file. The deploy command that runs after that, +at first will trigger templating process: filling your spec templates with variables, creating resource spec files. +That files become a targets for the provisioner module, which does attempts to create K8S resources. + +But in some cases, such as the intention to use your own templating engine or, probably, necessity to make specs +beforehand and to deploy them separately and later, there may be a need to divide the process into the separate steps: +1. Templating +2. Direct, `kubectl apply`-like provisioning without config.yaml context. + +For this reason, `k8s-handle render`, `k8s-handle apply`, `k8s-handle delete` commands are implemented. + +### Render + +`render` command is purposed for creating specs from templates without their subsequent deployment. + +Another purpose is to check the generation of the templates: previously, this functionality was achieved by using the +`--dry-run` optional flag. The support of `--dry-run` in `deploy` and `destroy` commands remains at this time for the +sake of backward compatibility but it's **discouraged** for the further usage. + +Just like with `deploy` command, `-s/--section` and `--tags`/`--skip-tags` targeting options are provided to make it +handy to render several specs. Connection parameters are not needed to be specified cause no k8s cluster availability +checks are performed. + +Templates directory path is taken from env `TEMPLATES_DIR` and equal to 'templates' by default. +Resources generated by this command can be obtained in directory that set in `TEMP_DIR` env variable +with default value '/tmp/k8s-handle'. Users that want to preserve generated templates might need to change this default +to avoid loss of the generated resources. + +``` +TEMP_DIR="/home/custom_dir" k8s-handle render -s staging +2019-02-15 14:44:44 INFO:k8s_handle.templating:Trying to generate file from template "service.yaml.j2" in "/home/custom_dir" +2019-02-15 14:44:44 INFO:k8s_handle.templating:File "/home/custom_dir/service.yaml" successfully generated +``` + +### Apply + +`apply` command with the `-r/--resource` required flag starts the process of provisioning of separate resource +spec to k8s. + +The value of `-r` key is considered as absolute path if it's started with slash. Otherwise, it's considered as +relative path from directory specified in `TEMP_DIR` env variable. + +No config.yaml-like file is required (and not taken into account even if exists). The connection parameters can be set +via `--use-kubeconfig` mode which is available and the most handy, or via the CLI/env flags and variables. +Options related to output and syncing, like `--sync-mode`, `--tries` and `--show-logs` are available as well. + +``` +$ k8s-handle apply -r /tmp/k8s-handle/service.yaml --use-kubeconfig +2019-02-15 14:22:58 INFO:k8s_handle:Default namespace "test" +2019-02-15 14:22:58 INFO:k8s_handle.k8s.resource:Using namespace "test" +2019-02-15 14:22:58 INFO:k8s_handle.k8s.resource:Service "k8s-handle-example" does not exist, create it + +``` + +### Delete +`delete` command with the `-r/--resource` required flag acts similarly to `destroy` command and does a try to delete + the directly specified resource from k8s if any. + +``` +$ k8s-handle delete -r service.yaml --use-kubeconfig + +2019-02-15 14:24:06 INFO:k8s_handle:Default namespace "test" +2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Using namespace "test" +2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Trying to delete Service "k8s-handle-example" +2019-02-15 14:24:06 INFO:k8s_handle.k8s.resource:Service "k8s-handle-example" deleted +``` \ No newline at end of file diff --git a/k8s_handle/__init__.py b/k8s_handle/__init__.py index c80fc38..aed0a37 100644 --- a/k8s_handle/__init__.py +++ b/k8s_handle/__init__.py @@ -15,19 +15,125 @@ from k8s_handle.k8s.deprecation_checker import ApiDeprecationChecker, DeprecationError from k8s_handle.k8s.resource import Provisioner, ProvisioningError +COMMAND_DEPLOY = 'deploy' +COMMAND_DESTROY = 'destroy' + log = logging.getLogger(__name__) logging.basicConfig(level=settings.LOG_LEVEL, format=settings.LOG_FORMAT, datefmt=settings.LOG_DATE_FORMAT) + +def handler_deploy(args): + _handler_deploy_destroy(args, COMMAND_DEPLOY) + + +def handler_destroy(args): + _handler_deploy_destroy(args, COMMAND_DESTROY) + + +def handler_apply(args): + _handler_apply_delete(args, COMMAND_DEPLOY) + + +def handler_delete(args): + _handler_apply_delete(args, COMMAND_DESTROY) + + +def handler_render(args): + context = config.load_context_section(args.get('section')) + templating.Renderer( + settings.TEMPLATES_DIR, + args.get('tags'), + args.get('skip_tags') + ).generate_by_context(context) + + +def _handler_deploy_destroy(args, command): + context = config.load_context_section(args.get('section')) + resources = templating.Renderer( + settings.TEMPLATES_DIR, + args.get('tags'), + args.get('skip_tags') + ).generate_by_context(context) + + if args.get('dry_run'): + return + + _handler_provision( + command, + resources, + config.PriorityEvaluator(args, context, os.environ), + args.get('use_kubeconfig'), + args.get('sync_mode'), + args.get('show_logs') + ) + + +def _handler_apply_delete(args, command): + _handler_provision( + command, + [os.path.join(settings.TEMP_DIR, args.get('resource'))], + config.PriorityEvaluator(args, {}, os.environ), + args.get('use_kubeconfig'), + args.get('sync_mode'), + args.get('show_logs') + ) + + +def _handler_provision(command, resources, priority_evaluator, use_kubeconfig, sync_mode, show_logs): + kubeconfig_namespace = None + + if priority_evaluator.environment_deprecated(): + log.warning("K8S_HOST and K8S_CA environment variables support is deprecated " + "and will be discontinued in the future. Use K8S_MASTER_URI and K8S_CA_BASE64 instead.") + + # INFO rvadim: https://github.com/kubernetes-client/python/issues/430#issuecomment-359483997 + if use_kubeconfig: + try: + load_kube_config() + kubeconfig_namespace = list_kube_config_contexts()[1].get('context').get('namespace') + except Exception as e: + raise RuntimeError(e) + else: + client.Configuration.set_default(priority_evaluator.k8s_client_configuration()) + + settings.K8S_NAMESPACE = priority_evaluator.k8s_namespace_default(kubeconfig_namespace) + log.info('Default namespace "{}"'.format(settings.K8S_NAMESPACE)) + + if not settings.K8S_NAMESPACE: + log.info("Default namespace is not set. " + "This may lead to provisioning error, if namespace is not set for each resource.") + + d = ApiDeprecationChecker(client.VersionApi().get_code().git_version[1:]) + p = Provisioner(command, sync_mode, show_logs) + + for resource in resources: + d.run(resource) + + for resource in resources: + p.run(resource) + + parser = argparse.ArgumentParser(description='CLI utility generate k8s resources by templates and apply it to cluster') subparsers = parser.add_subparsers(dest="command") subparsers.required = True -parser_target = argparse.ArgumentParser(add_help=False) -parser_target.add_argument('-s', '--section', required=True, type=str, help='Section to deploy from config file') -parser_target.add_argument('-c', '--config', required=False, help='Config file, default: config.yaml') +parser_target_config = argparse.ArgumentParser(add_help=False) +parser_target_config.add_argument('-s', '--section', required=True, type=str, help='Section to deploy from config file') +parser_target_config.add_argument('-c', '--config', required=False, help='Config file, default: config.yaml') +parser_target_config.add_argument('--tags', action='append', required=False, + help='Only use templates tagged with these values') +parser_target_config.add_argument('--skip-tags', action='append', required=False, + help='Only use templates whose tags do not match these values') + +parser_target_resource = argparse.ArgumentParser(add_help=False) +parser_target_resource.add_argument('-r', '--resource', required=True, type=str, + help='Resource spec path, absolute (started with slash) or relative from TEMP_DIR') + +parser_deprecated = argparse.ArgumentParser(add_help=False) +parser_deprecated.add_argument('--dry-run', required=False, action='store_true', + help='Don\'t run kubectl commands. Deprecated, use "k8s-handle template" instead') parser_provisioning = argparse.ArgumentParser(add_help=False) -parser_provisioning.add_argument('--dry-run', required=False, action='store_true', help='Don\'t run kubectl commands') parser_provisioning.add_argument('--sync-mode', action='store_true', required=False, default=False, help='Turn on sync mode and wait deployment ending') parser_provisioning.add_argument('--tries', type=int, required=False, default=360, @@ -40,23 +146,39 @@ help='Try to use kube config') parser_provisioning.add_argument('--k8s-handle-debug', action='store_true', required=False, help='Show K8S client debug messages') -parser_provisioning.add_argument('--tags', action='append', required=False, - help='Only use templates tagged with these values') -parser_provisioning.add_argument('--skip-tags', action='append', required=False, - help='Only use templates whose tags do not match these values') + +parser_logs = argparse.ArgumentParser(add_help=False) +parser_logs.add_argument('--show-logs', action='store_true', required=False, default=False, help='Show logs for jobs') +parser_logs.add_argument('--tail-lines', type=int, required=False, help='Lines of recent log file to display') arguments_connection = parser_provisioning.add_argument_group() arguments_connection.add_argument('--k8s-master-uri', required=False, help='K8S master to connect to') arguments_connection.add_argument('--k8s-ca-base64', required=False, help='base64-encoded K8S certificate authority') arguments_connection.add_argument('--k8s-token', required=False, help='K8S token to use') -parser_deploy = subparsers.add_parser('deploy', parents=[parser_provisioning, parser_target], - help='Sub command for deploy app') -parser_deploy.add_argument('--show-logs', action='store_true', required=False, default=False, help='Show logs for jobs') -parser_deploy.add_argument('--tail-lines', type=int, required=False, help='Lines of recent log file to display') +parser_deploy = subparsers.add_parser( + 'deploy', + parents=[parser_provisioning, parser_target_config, parser_logs, parser_deprecated], + help='Do attempt to create specs from templates and deploy K8S resources of the selected section') +parser_deploy.set_defaults(func=handler_deploy) + +parser_apply = subparsers.add_parser('apply', parents=[parser_provisioning, parser_target_resource, parser_logs], + help='Do attempt to deploy K8S resource from the existing spec') +parser_apply.set_defaults(func=handler_apply) + +parser_destroy = subparsers.add_parser('destroy', + parents=[parser_provisioning, parser_target_config, parser_deprecated], + help='Do attempt to destroy K8S resources of the selected section') +parser_destroy.set_defaults(func=handler_destroy) -parser_destroy = subparsers.add_parser('destroy', parents=[parser_provisioning, parser_target], - help='Sub command for destroy app') +parser_delete = subparsers.add_parser('delete', parents=[parser_provisioning, parser_target_resource], + help='Do attempt to destroy K8S resource from the existing spec') +parser_delete.set_defaults(func=handler_delete) + +parser_template = subparsers.add_parser('render', parents=[parser_target_config], + help='Make resources from the template and config. ' + 'Created resources will be placed into the TEMP_DIR') +parser_template.set_defaults(func=handler_render) def main(): @@ -83,57 +205,17 @@ def main(): log.warning("Explicit true/false arguments to --sync-mode and --dry-run keys are deprecated " "and will be discontinued in the future. Use these keys without arguments instead.") - args = vars(args) - kubeconfig_namespace = None - settings.CHECK_STATUS_TRIES = args.get('tries') - settings.CHECK_DAEMONSET_STATUS_TRIES = args.get('tries') - settings.CHECK_STATUS_TIMEOUT = args.get('retry_delay') - settings.CHECK_DAEMONSET_STATUS_TIMEOUT = args.get('retry_delay') - settings.GET_ENVIRON_STRICT = args.get('strict') - settings.ONLY_TAGS = args.get('tags') - settings.SKIP_TAGS = args.get('skip_tags') - settings.COUNT_LOG_LINES = args.get('tail_lines') - settings.CONFIG_FILE = args.get('config') or settings.CONFIG_FILE + args_dict = vars(args) + settings.CHECK_STATUS_TRIES = args_dict.get('tries') + settings.CHECK_DAEMONSET_STATUS_TRIES = args_dict.get('tries') + settings.CHECK_STATUS_TIMEOUT = args_dict.get('retry_delay') + settings.CHECK_DAEMONSET_STATUS_TIMEOUT = args_dict.get('retry_delay') + settings.GET_ENVIRON_STRICT = args_dict.get('strict') + settings.COUNT_LOG_LINES = args_dict.get('tail_lines') + settings.CONFIG_FILE = args_dict.get('config') or settings.CONFIG_FILE try: - context = config.load_context_section(args['section']) - render = templating.Renderer(settings.TEMPLATES_DIR) - resources = render.generate_by_context(context) - evaluator = config.PriorityEvaluator(args, context, os.environ) - - if evaluator.environment_deprecated(): - log.warning("K8S_HOST and K8S_CA environment variables support is deprecated " - "and will be discontinued in the future. Use K8S_MASTER_URI and K8S_CA_BASE64 instead.") - - if args.get('dry_run'): - return - - # INFO rvadim: https://github.com/kubernetes-client/python/issues/430#issuecomment-359483997 - if args.get('use_kubeconfig'): - try: - load_kube_config() - kubeconfig_namespace = list_kube_config_contexts()[1].get('context').get('namespace') - except Exception as e: - raise RuntimeError(e) - else: - client.Configuration.set_default(evaluator.k8s_client_configuration()) - - settings.K8S_NAMESPACE = evaluator.k8s_namespace_default(kubeconfig_namespace) - log.info('Default namespace "{}"'.format(settings.K8S_NAMESPACE)) - - if not settings.K8S_NAMESPACE: - log.info("Default namespace is not set. " - "This may lead to provisioning error, if namespace is not set for each resource.") - - p = Provisioner(args['command'], args.get('sync_mode'), args.get('show_logs')) - d = ApiDeprecationChecker(client.VersionApi().get_code().git_version[1:]) - - for resource in resources: - d.run(resource) - - for resource in resources: - p.run(resource) - + args.func(args_dict) except templating.TemplateRenderingError as e: log.error('Template generation error: {}'.format(e)) sys.exit(1) diff --git a/k8s_handle/settings.py b/k8s_handle/settings.py index 547524e..c79b923 100644 --- a/k8s_handle/settings.py +++ b/k8s_handle/settings.py @@ -30,6 +30,3 @@ COUNT_LOG_LINES = None GET_ENVIRON_STRICT = False - -ONLY_TAGS = None -SKIP_TAGS = None diff --git a/k8s_handle/templating.py b/k8s_handle/templating.py index 73fd0aa..bd39aad 100644 --- a/k8s_handle/templating.py +++ b/k8s_handle/templating.py @@ -1,13 +1,15 @@ -import os -import glob import base64 +import glob import itertools import logging -import yaml -from k8s_handle import settings +import os from hashlib import sha256 + +import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined -from jinja2.exceptions import TemplateNotFound, UndefinedError, TemplateSyntaxError +from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError, UndefinedError + +from k8s_handle import settings class TemplateRenderingError(Exception): @@ -18,27 +20,30 @@ class TemplateRenderingError(Exception): def get_template_contexts(file_path): - with open(file_path) as f: - try: - contexts = yaml.safe_load_all(f.read()) - except Exception as e: - raise RuntimeError('Unable to load yaml file: {}, {}'.format(file_path, e)) - - for context in contexts: - if context is None: - continue # Skip empty YAML documents - if 'kind' not in context or context['kind'] is None: - raise RuntimeError('Field "kind" not found (or empty) in file "{}"'.format(file_path)) - if 'metadata' not in context or context['metadata'] is None: - raise RuntimeError('Field "metadata" not found (or empty) in file "{}"'.format(file_path)) - if 'name' not in context['metadata'] or context['metadata']['name'] is None: - raise RuntimeError('Field "metadata->name" not found (or empty) in file "{}"'.format(file_path)) - if 'spec' in context: - # INFO: Set replicas = 1 by default for replaces cases in Deployment and StatefulSet - if 'replicas' not in context['spec'] or context['spec']['replicas'] is None: - if context['kind'] in ['Deployment', 'StatefulSet']: - context['spec']['replicas'] = 1 - yield context + try: + with open(file_path) as f: + try: + contexts = yaml.safe_load_all(f.read()) + except Exception as e: + raise RuntimeError('Unable to load yaml file: {}, {}'.format(file_path, e)) + + for context in contexts: + if context is None: + continue # Skip empty YAML documents + if 'kind' not in context or context['kind'] is None: + raise RuntimeError('Field "kind" not found (or empty) in file "{}"'.format(file_path)) + if 'metadata' not in context or context['metadata'] is None: + raise RuntimeError('Field "metadata" not found (or empty) in file "{}"'.format(file_path)) + if 'name' not in context['metadata'] or context['metadata']['name'] is None: + raise RuntimeError('Field "metadata->name" not found (or empty) in file "{}"'.format(file_path)) + if 'spec' in context: + # INFO: Set replicas = 1 by default for replaces cases in Deployment and StatefulSet + if 'replicas' not in context['spec'] or context['spec']['replicas'] is None: + if context['kind'] in ['Deployment', 'StatefulSet']: + context['spec']['replicas'] = 1 + yield context + except FileNotFoundError as e: + raise RuntimeError(e) def b64decode(string): @@ -66,6 +71,7 @@ def include_file(path): with open(file_path, 'r') as f: output.append(f.read()) return '\n'.join(output) + env = Environment( undefined=StrictUndefined, loader=FileSystemLoader([templates_dir])) @@ -79,17 +85,11 @@ def include_file(path): return env -def _create_dir(dir): - try: - os.makedirs(dir) - except os.error as e: - log.debug(e) - pass - - class Renderer: - def __init__(self, templates_dir): + def __init__(self, templates_dir, tags=None, tags_skip=None): self._templates_dir = templates_dir + self._tags = tags + self._tags_skip = tags_skip self._env = get_env(self._templates_dir) def generate_by_context(self, context): @@ -103,9 +103,7 @@ def generate_by_context(self, context): return templates = filter( - lambda i: self._evaluate_tags(self._get_template_tags(i), - settings.ONLY_TAGS, - settings.SKIP_TAGS), + lambda i: self._evaluate_tags(self._get_template_tags(i), self._tags, self._tags_skip), templates ) @@ -121,41 +119,9 @@ def generate_by_context(self, context): raise TemplateRenderingError('Unable to render {}, due to: {}'.format(template, e)) return output - def _get_template_tags(self, template): - if 'tags' not in template: - return set(['untagged']) - - tags = template['tags'] - - if isinstance(tags, list): - tags = set([i for i, _ in itertools.groupby(tags)]) - elif isinstance(tags, str): - tags = set(tags.split(',')) - else: - raise TypeError('Unable to parse tags of "{}" template: unexpected type {}'.format(template, - type(tags))) - - return tags - - def _evaluate_tags(self, tags, only_tags, skip_tags): - skip = False - - if only_tags: - if tags.isdisjoint(only_tags): - skip = True - - if skip_tags: - if skip: - pass # we decided to skip template already - elif not tags.isdisjoint(skip_tags): - skip = True - - return not skip - - def _generate_file(self, item, dir, context): - _create_dir(dir) + def _generate_file(self, item, directory, context): try: - log.info('Trying to generate file from template "{}" in "{}"'.format(item['template'], dir)) + log.info('Trying to generate file from template "{}" in "{}"'.format(item['template'], directory)) template = self._env.get_template(item['template']) except TemplateNotFound as e: log.info('Templates path: {}, available templates:{}'.format(self._templates_dir, @@ -163,10 +129,39 @@ def _generate_file(self, item, dir, context): raise e except KeyError: raise RuntimeError('Templates section doesn\'t have any template items') + new_name = item['template'].replace('.j2', '') - path = os.path.join(dir, new_name) - if not os.path.exists(os.path.dirname(path)): - _create_dir(os.path.dirname(path)) - with open(path, 'w+') as f: - f.write(template.render(context)) + path = os.path.join(directory, new_name) + + try: + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + with open(path, 'w+') as f: + f.write(template.render(context)) + + except TemplateRenderingError: + raise + except (FileNotFoundError, PermissionError) as e: + raise RuntimeError(e) + return path + + @staticmethod + def _get_template_tags(template): + if 'tags' not in template: + return {'untagged'} + + tags = template['tags'] + + if isinstance(tags, list): + return set([i for i, _ in itertools.groupby(tags)]) + + if isinstance(tags, str): + return set(tags.split(',')) + + raise TypeError('Unable to parse tags of "{}" template: unexpected type {}'.format(template, type(tags))) + + @staticmethod + def _evaluate_tags(tags, only_tags, skip_tags): + return tags.isdisjoint(skip_tags or []) and not tags.isdisjoint(only_tags or tags) diff --git a/tests/test_templating.py b/tests/test_templating.py index 336335d..2328ac1 100644 --- a/tests/test_templating.py +++ b/tests/test_templating.py @@ -105,11 +105,11 @@ def test_io_2709(self): def test_evaluate_tags(self): r = templating.Renderer(os.path.join(os.path.dirname(__file__), 'templates_tests')) tags = {'tag1', 'tag2', 'tag3'} - assert r._evaluate_tags(tags, only_tags=['tag1'], skip_tags=None) is True - assert r._evaluate_tags(tags, only_tags=['tag4'], skip_tags=None) is False - assert r._evaluate_tags(tags, only_tags=['tag1'], skip_tags=['tag1']) is False - assert r._evaluate_tags(tags, only_tags=None, skip_tags=['tag1']) is False - assert r._evaluate_tags(tags, only_tags=None, skip_tags=['tag4']) is True + self.assertTrue(r._evaluate_tags(tags, only_tags=['tag1'], skip_tags=None)) + self.assertFalse(r._evaluate_tags(tags, only_tags=['tag4'], skip_tags=None)) + self.assertFalse(r._evaluate_tags(tags, only_tags=['tag1'], skip_tags=['tag1'])) + self.assertFalse(r._evaluate_tags(tags, only_tags=None, skip_tags=['tag1'])) + self.assertTrue(r._evaluate_tags(tags, only_tags=None, skip_tags=['tag4'])) def test_get_template_tags(self): r = templating.Renderer(os.path.join(os.path.dirname(__file__), 'templates_tests')) @@ -117,10 +117,10 @@ def test_get_template_tags(self): template_2 = {'template': 'template.yaml.j2', 'tags': 'tag1,tag2,tag3'} template_3 = {'template': 'template.yaml.j2', 'tags': ['tag1']} template_4 = {'template': 'template.yaml.j2', 'tags': 'tag1'} - assert r._get_template_tags(template_1) == {'tag1', 'tag2', 'tag3'} - assert r._get_template_tags(template_2) == {'tag1', 'tag2', 'tag3'} - assert r._get_template_tags(template_3) == {'tag1'} - assert r._get_template_tags(template_4) == {'tag1'} + self.assertEqual(r._get_template_tags(template_1), {'tag1', 'tag2', 'tag3'}) + self.assertEqual(r._get_template_tags(template_2), {'tag1', 'tag2', 'tag3'}) + self.assertEqual(r._get_template_tags(template_3), {'tag1'}) + self.assertEqual(r._get_template_tags(template_4), {'tag1'}) def test_get_template_tags_unexpected_type(self): r = templating.Renderer(os.path.join(os.path.dirname(__file__), 'templates_tests'))