Add coverage testing

fatbasstard · Dec 12, 2018 · ad5f711 · ad5f711
1 parent dccb3de
commit ad5f711
Show file tree

Hide file tree

Showing 118 changed files with 293 additions and 281 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+branch = True
+include =
+  *src/cfnlint/*
diff --git a/.travis.yml b/.travis.yml
@@ -23,7 +23,7 @@ jobs:
   - python: 3.5
     env: TOXENV=py35
   - python: 3.6
-    env: TOXENV=py36
+    env: TOXENV=py36,codecov
   - python: 3.7
     env: TOXENV=py37
   - stage: Integration

diff --git a/docs/getting_started/rules.md b/docs/getting_started/rules.md
@@ -131,7 +131,7 @@ class TestMyNewRule(BaseRuleTestCase):
 
     def test_file_negative(self):
         """Test failure"""
-        self.helper_file_negative('fixtures/templates/bad/mynewrule.yaml', 1) # Amount of expected matches
+        self.helper_file_negative('test/fixtures/templates/bad/mynewrule.yaml', 1) # Amount of expected matches
 ```
 
 As you can see `test_file_negative()` in this unit test makes specific use of a CloudFormation template from the Fixtures folder. It's important to provide examples of templates which pass your new rule, and also templates which generate the expected warning/error.

diff --git a/test/integration/test_good_templates.py b/test/integration/test_good_templates.py
@@ -32,35 +32,35 @@ def setUp(self):
 
         self.filenames = {
             'generic': {
-                "filename": 'fixtures/templates/good/generic.yaml',
+                "filename": 'test/fixtures/templates/good/generic.yaml',
                 "failures": 0
             },
             'minimal': {
-                "filename": 'fixtures/templates/good/minimal.yaml',
+                "filename": 'test/fixtures/templates/good/minimal.yaml',
                 "failures": 0
             },
             'transform': {
-                "filename": 'fixtures/templates/good/transform.yaml',
+                "filename": 'test/fixtures/templates/good/transform.yaml',
                 "failures": 0
             },
             'conditions': {
-                "filename": 'fixtures/templates/good/conditions.yaml',
+                "filename": 'test/fixtures/templates/good/conditions.yaml',
                 "failures": 0
             },
             'resources_codepipeline': {
-                'filename': 'fixtures/templates/good/resources_codepipeline.yaml',
+                'filename': 'test/fixtures/templates/good/resources_codepipeline.yaml',
                 'failures': 0
             },
             'transform_serverless_api': {
-                'filename': 'fixtures/templates/good/transform_serverless_api.yaml',
+                'filename': 'test/fixtures/templates/good/transform_serverless_api.yaml',
                 'failures': 0
             },
             'transform_serverless_function': {
-                'filename': 'fixtures/templates/good/transform_serverless_function.yaml',
+                'filename': 'test/fixtures/templates/good/transform_serverless_function.yaml',
                 'failures': 0
             },
             'transform_serverless_globals': {
-                'filename': 'fixtures/templates/good/transform_serverless_globals.yaml',
+                'filename': 'test/fixtures/templates/good/transform_serverless_globals.yaml',
                 'failures': 0
             }
         }

diff --git a/test/integration/test_quickstart_templates.py b/test/integration/test_quickstart_templates.py
@@ -33,52 +33,52 @@ def setUp(self):
 
         self.filenames = {
             'config_rule': {
-                "filename": 'fixtures/templates/public/lambda-poller.yaml',
+                "filename": 'test/fixtures/templates/public/lambda-poller.yaml',
                 "failures": 0
             },
             'watchmaker': {
-                "filename": 'fixtures/templates/public/watchmaker.json',
+                "filename": 'test/fixtures/templates/public/watchmaker.json',
                 "failures": 0
             },
             'nist_high_master': {
-                'filename': 'fixtures/templates/quickstart/nist_high_master.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_high_master.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_high_master.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_high_master.json'
             },
             'nist_application': {
-                'filename': 'fixtures/templates/quickstart/nist_application.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_application.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_application.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_application.json'
             },
             'nist_config_rules': {
-                'filename': 'fixtures/templates/quickstart/nist_config_rules.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_config_rules.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_config_rules.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_config_rules.json'
             },
             'nist_iam': {
-                'filename': 'fixtures/templates/quickstart/nist_iam.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_iam.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_iam.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_iam.json'
             },
             'nist_logging': {
-                'filename': 'fixtures/templates/quickstart/nist_logging.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_logging.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_logging.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_logging.json'
             },
             'nist_vpc_management': {
-                'filename': 'fixtures/templates/quickstart/nist_vpc_management.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_vpc_management.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_vpc_management.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_vpc_management.json'
             },
             'nist_vpc_production': {
-                'filename': 'fixtures/templates/quickstart/nist_vpc_production.yaml',
-                'results_filename': 'fixtures/results/quickstart/nist_vpc_production.json'
+                'filename': 'test/fixtures/templates/quickstart/nist_vpc_production.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/nist_vpc_production.json'
             },
             'openshift_master': {
-                'filename': 'fixtures/templates/quickstart/openshift_master.yaml',
+                'filename': 'test/fixtures/templates/quickstart/openshift_master.yaml',
                 'failures': 0
             },
             'openshift': {
-                'filename': 'fixtures/templates/quickstart/openshift.yaml',
-                'results_filename': 'fixtures/results/quickstart/openshift.json'
+                'filename': 'test/fixtures/templates/quickstart/openshift.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/openshift.json'
             },
             'cis_benchmark': {
-                'filename': 'fixtures/templates/quickstart/cis_benchmark.yaml',
-                'results_filename': 'fixtures/results/quickstart/cis_benchmark.json'
+                'filename': 'test/fixtures/templates/quickstart/cis_benchmark.yaml',
+                'results_filename': 'test/fixtures/results/quickstart/cis_benchmark.json'
             }
         }
 

diff --git a/test/module/cfn_json/test_cfn_json.py b/test/module/cfn_json/test_cfn_json.py
@@ -35,27 +35,27 @@ def setUp(self):
 
         self.filenames = {
             "config_rule": {
-                "filename": 'fixtures/templates/quickstart/config-rules.json',
+                "filename": 'test/fixtures/templates/quickstart/config-rules.json',
                 "failures": 4
             },
             "iam": {
-                "filename": 'fixtures/templates/quickstart/iam.json',
+                "filename": 'test/fixtures/templates/quickstart/iam.json',
                 "failures": 4
             },
             "nat_instance": {
-                "filename": 'fixtures/templates/quickstart/nat-instance.json',
+                "filename": 'test/fixtures/templates/quickstart/nat-instance.json',
                 "failures": 2
             },
             "vpc_management": {
-                "filename": 'fixtures/templates/quickstart/vpc-management.json',
+                "filename": 'test/fixtures/templates/quickstart/vpc-management.json',
                 "failures": 35
             },
             "vpc": {
-                "filename": 'fixtures/templates/quickstart/vpc.json',
+                "filename": 'test/fixtures/templates/quickstart/vpc.json',
                 "failures": 40
             },
             "poller": {
-                "filename": 'fixtures/templates/public/lambda-poller.json',
+                "filename": 'test/fixtures/templates/public/lambda-poller.json',
                 "failures": 0
             }
         }
@@ -92,7 +92,7 @@ def test_success_parse_stdin(self):
     def test_fail_run(self):
         """Test failure run"""
 
-        filename = 'fixtures/templates/bad/json_parse.json'
+        filename = 'test/fixtures/templates/bad/json_parse.json'
 
         try:
             template = cfnlint.decode.cfn_json.load(filename)

diff --git a/test/module/cfn_yaml/test_yaml.py b/test/module/cfn_yaml/test_yaml.py
@@ -34,11 +34,11 @@ def setUp(self):
 
         self.filenames = {
             "config_rule": {
-                "filename": 'fixtures/templates/public/lambda-poller.yaml',
+                "filename": 'test/fixtures/templates/public/lambda-poller.yaml',
                 "failures": 0
             },
             "generic_bad": {
-                "filename": 'fixtures/templates/bad/generic.yaml',
+                "filename": 'test/fixtures/templates/bad/generic.yaml',
                 "failures": 35
             }
         }

diff --git a/test/module/config/test_config_mixin.py b/test/module/config/test_config_mixin.py
@@ -86,15 +86,15 @@ def test_config_expand_paths(self, yaml_mock):
         """ Test precedence in  """
 
         yaml_mock.side_effect = [
-            {'templates': ['fixtures/templates/public/*.yaml']},
+            {'templates': ['test/fixtures/templates/public/*.yaml']},
             {}
         ]
         config = cfnlint.config.ConfigMixIn([])
 
         # test defaults
         self.assertEqual(config.templates, [
-            'fixtures/templates/public/lambda-poller.yaml',
-            'fixtures/templates/public/rds-cluster.yaml'])
+            'test/fixtures/templates/public/lambda-poller.yaml',
+            'test/fixtures/templates/public/rds-cluster.yaml'])
 
     @patch('cfnlint.config.ConfigFileArgs._read_config', create=True)
     def test_config_expand_paths_failure(self, yaml_mock):

diff --git a/test/module/core/test_run_checks.py b/test/module/core/test_run_checks.py
@@ -25,7 +25,7 @@ class TestRunChecks(BaseTestCase):
     def test_good_template(self):
         """Test success run"""
 
-        filename = 'fixtures/templates/good/generic.yaml'
+        filename = 'test/fixtures/templates/good/generic.yaml'
         (args, filenames, _) = cfnlint.core.get_args_filenames([
             '--template', filename])
 
@@ -41,7 +41,7 @@ def test_good_template(self):
     def test_bad_template(self):
         """Test bad template"""
 
-        filename = 'fixtures/templates/quickstart/nat-instance.json'
+        filename = 'test/fixtures/templates/quickstart/nat-instance.json'
         (args, filenames, _) = cfnlint.core.get_args_filenames([
             '--template', filename])
         results = []
@@ -56,7 +56,7 @@ def test_bad_template(self):
 
     def test_bad_region(self):
         """Test bad region"""
-        filename = 'fixtures/templates/good/generic.yaml'
+        filename = 'test/fixtures/templates/good/generic.yaml'
         (args, filenames, _) = cfnlint.core.get_args_filenames(['--template', filename])
         (template, rules, _) = cfnlint.core.get_template_rules(filename, args)
         err = None

diff --git a/test/module/core/test_run_cli.py b/test/module/core/test_run_cli.py
@@ -33,7 +33,7 @@ def tearDown(self):
     def test_template_not_found(self):
         """Test template not found"""
 
-        filename = 'fixtures/templates/good/core/not_found.yaml'
+        filename = 'test/fixtures/templates/good/core/not_found.yaml'
 
         (args, filenames, _) = cfnlint.core.get_args_filenames(
             ['--template', filename, '--ignore_bad_template'])
@@ -43,7 +43,7 @@ def test_template_not_found(self):
 
     def test_template_invalid_yaml(self):
         """Test template not found"""
-        filename = 'fixtures/templates/bad/core/config_invalid_yaml.yaml'
+        filename = 'test/fixtures/templates/bad/core/config_invalid_yaml.yaml'
 
         (args, filenames, _) = cfnlint.core.get_args_filenames(
             ['--template', filename])
@@ -53,7 +53,7 @@ def test_template_invalid_yaml(self):
 
     def test_template_invalid_json(self):
         """Test template not found"""
-        filename = 'fixtures/templates/bad/core/config_invalid_json.json'
+        filename = 'test/fixtures/templates/bad/core/config_invalid_json.json'
 
         (args, filenames, _) = cfnlint.core.get_args_filenames(
             ['--template', filename])
@@ -63,7 +63,7 @@ def test_template_invalid_json(self):
 
     def test_template_invalid_yaml_ignore(self):
         """Test template not found"""
-        filename = 'fixtures/templates/bad/core/config_invalid_yaml.yaml'
+        filename = 'test/fixtures/templates/bad/core/config_invalid_yaml.yaml'
 
         (args, filenames, _) = cfnlint.core.get_args_filenames(
             ['--template', filename, '--ignore-bad-template'])
@@ -73,7 +73,7 @@ def test_template_invalid_yaml_ignore(self):
 
     def test_template_invalid_json_ignore(self):
         """Test template not found"""
-        filename = 'fixtures/templates/bad/core/config_invalid_json.json'
+        filename = 'test/fixtures/templates/bad/core/config_invalid_json.json'
 
         (args, filenames, _) = cfnlint.core.get_args_filenames(
             ['--template', filename, '--ignore-bad-template'])
@@ -83,7 +83,7 @@ def test_template_invalid_json_ignore(self):
 
     def test_template_config(self):
         """Test template config"""
-        filename = 'fixtures/templates/good/core/config_parameters.yaml'
+        filename = 'test/fixtures/templates/good/core/config_parameters.yaml'
         (args, _, _,) = cfnlint.core.get_args_filenames([
             '--template', filename, '--ignore-bad-template'])
 
@@ -97,13 +97,13 @@ def test_template_config(self):
         self.assertEqual(args.debug, False)
         self.assertEqual(args.override_spec, None)
         self.assertEqual(args.regions, ['us-east-1'])
-        self.assertEqual(args.templates, ['fixtures/templates/good/core/config_parameters.yaml'])
+        self.assertEqual(args.templates, ['test/fixtures/templates/good/core/config_parameters.yaml'])
         self.assertEqual(args.update_documentation, False)
         self.assertEqual(args.update_specs, False)
 
     def test_positional_template_parameters(self):
         """Test overriding parameters"""
-        filename = 'fixtures/templates/good/core/config_parameters.yaml'
+        filename = 'test/fixtures/templates/good/core/config_parameters.yaml'
         (args, _, _) = cfnlint.core.get_args_filenames([
             filename, '--ignore-bad-template',
             '--ignore-checks', 'E0000'])
@@ -117,13 +117,13 @@ def test_positional_template_parameters(self):
         self.assertEqual(args.debug, False)
         self.assertEqual(args.override_spec, None)
         self.assertEqual(args.regions, ['us-east-1'])
-        self.assertEqual(args.templates, ['fixtures/templates/good/core/config_parameters.yaml'])
+        self.assertEqual(args.templates, ['test/fixtures/templates/good/core/config_parameters.yaml'])
         self.assertEqual(args.update_documentation, False)
         self.assertEqual(args.update_specs, False)
 
     def test_override_parameters(self):
         """Test overriding parameters"""
-        filename = 'fixtures/templates/good/core/config_parameters.yaml'
+        filename = 'test/fixtures/templates/good/core/config_parameters.yaml'
         (args, _, _) = cfnlint.core.get_args_filenames([
             '--template', filename, '--ignore-bad-template',
             '--ignore-checks', 'E0000'])
@@ -137,14 +137,14 @@ def test_override_parameters(self):
         self.assertEqual(args.debug, False)
         self.assertEqual(args.override_spec, None)
         self.assertEqual(args.regions, ['us-east-1'])
-        self.assertEqual(args.templates, ['fixtures/templates/good/core/config_parameters.yaml'])
+        self.assertEqual(args.templates, ['test/fixtures/templates/good/core/config_parameters.yaml'])
         self.assertEqual(args.update_documentation, False)
         self.assertEqual(args.update_specs, False)
 
     def test_bad_config(self):
         """ Test bad formatting in config"""
 
-        filename = 'fixtures/templates/bad/core/config_parameters.yaml'
+        filename = 'test/fixtures/templates/bad/core/config_parameters.yaml'
         (args, _, _) = cfnlint.core.get_args_filenames([
             '--template', filename, '--ignore-bad-template'])
 

diff --git a/test/module/override/test_complete.py b/test/module/override/test_complete.py
@@ -37,9 +37,9 @@ def tearDown(self):
 
     def test_success_run(self):
         """Success test"""
-        filename = 'fixtures/templates/good/override/complete.yaml'
+        filename = 'test/fixtures/templates/good/override/complete.yaml'
         template = self.load_template(filename)
-        with open('fixtures/templates/override_spec/complete.json') as fp:
+        with open('test/fixtures/templates/override_spec/complete.json') as fp:
             custom_spec = json.load(fp)
 
         cfnlint.helpers.set_specs(custom_spec)
@@ -49,10 +49,10 @@ def test_success_run(self):
 
     def test_fail_run(self):
         """Failure test required"""
-        filename = 'fixtures/templates/bad/override/complete.yaml'
+        filename = 'test/fixtures/templates/bad/override/complete.yaml'
         template = self.load_template(filename)
 
-        with open('fixtures/templates/override_spec/complete.json') as fp:
+        with open('test/fixtures/templates/override_spec/complete.json') as fp:
             custom_spec = json.load(fp)
         cfnlint.helpers.set_specs(custom_spec)
 

diff --git a/test/module/override/test_exclude.py b/test/module/override/test_exclude.py
@@ -35,9 +35,9 @@ def tearDown(self):
 
     def test_success_run(self):
         """Success test"""
-        filename = 'fixtures/templates/good/generic.yaml'
+        filename = 'test/fixtures/templates/good/generic.yaml'
         template = self.load_template(filename)
-        with open('fixtures/templates/override_spec/exclude.json') as fp:
+        with open('test/fixtures/templates/override_spec/exclude.json') as fp:
             custom_spec = json.load(fp)
 
         cfnlint.helpers.set_specs(custom_spec)
@@ -47,10 +47,10 @@ def test_success_run(self):
 
     def test_fail_run(self):
         """Failure test required"""
-        filename = 'fixtures/templates/bad/override/exclude.yaml'
+        filename = 'test/fixtures/templates/bad/override/exclude.yaml'
         template = self.load_template(filename)
 
-        with open('fixtures/templates/override_spec/exclude.json') as fp:
+        with open('test/fixtures/templates/override_spec/exclude.json') as fp:
             custom_spec = json.load(fp)
         cfnlint.helpers.set_specs(custom_spec)