add option to ignore duplicate includes

changelog/67081.added.md

@@ -0,0 +1 @@
+Add option to ignore duplicate includes

salt/config/__init__.py

@@ -790,6 +790,8 @@ def _gather_buffer_space():
         "jinja_trim_blocks": bool,
         # Cache minion ID to file
         "minion_id_caching": bool,
+        # Allow duplicate includes
+        "allow_duplicate_includes": bool,
         # Always generate minion id in lowercase.
         "minion_id_lowercase": bool,
         # Remove either a single domain (foo.org), or all (True) from a generated minion id.
@@ -1271,6 +1273,7 @@ def _gather_buffer_space():
         "grains_refresh_every": 0,
         "minion_id_caching": True,
         "minion_id_lowercase": False,
+        "allow_duplicate_includes": False,
         "minion_id_remove_domain": False,
         "keysize": 2048,
         "transport": "zeromq",

salt/renderers/yaml.py

@@ -34,7 +34,9 @@ def yaml_loader(*args):
             yamlloader = yamlloader_old
         else:
             yamlloader = yamlloader_new
-        return yamlloader.SaltYamlSafeLoader(*args, dictclass=OrderedDict)
+        return yamlloader.SaltYamlSafeLoader(
+            *args, dictclass=OrderedDict, opts=__opts__
+        )
 
     return yaml_loader
 

salt/utils/yamlloader.py

@@ -23,8 +23,9 @@ class SaltYamlSafeLoader(BaseLoader):
     to make things like sls file more intuitive.
     """
 
-    def __init__(self, stream, dictclass=dict):
+    def __init__(self, stream, dictclass=dict, opts=None):
         super().__init__(stream)
+        self.__opts__ = opts or {}
         if dictclass is not dict:
             # then assume ordered dict and use it for both !map and !omap
             self.add_constructor("tag:yaml.org,2002:map", type(self).construct_yaml_map)
@@ -76,6 +77,11 @@ def construct_mapping(self, node, deep=False):
                 )
             value = self.construct_object(value_node, deep=deep)
             if key in mapping:
+                if key == "include" and self.__opts__.get(
+                    "allow_duplicate_includes", False
+                ):
+                    continue
+
                 raise ConstructorError(
                     context,
                     node.start_mark,

salt/utils/yamlloader_old.py

@@ -28,8 +28,9 @@ class SaltYamlSafeLoader(yaml.SafeLoader):
     to make things like sls file more intuitive.
     """
 
-    def __init__(self, stream, dictclass=dict):
+    def __init__(self, stream, dictclass=dict, opts=None):
         super().__init__(stream)
+        self.__opts__ = opts or {}
         if dictclass is not dict:
             # then assume ordered dict and use it for both !map and !omap
             self.add_constructor("tag:yaml.org,2002:map", type(self).construct_yaml_map)
@@ -81,6 +82,11 @@ def construct_mapping(self, node, deep=False):
                 )
             value = self.construct_object(value_node, deep=deep)
             if key in mapping:
+                if key == "include" and self.__opts__.get(
+                    "allow_duplicate_includes", False
+                ):
+                    continue
+
                 raise ConstructorError(
                     context,
                     node.start_mark,

tests/unit/utils/test_yamldumper.py

@@ -0,0 +1,111 @@
+"""
+    Unit tests for salt.utils.yamldumper
+"""
+
+import pytest
+from yaml.constructor import ConstructorError
+
+import salt.utils.yamldumper
+from tests.support.unit import TestCase
+
+
+class YamlDumperTestCase(TestCase):
+    """
+    TestCase for salt.utils.yamldumper module
+    """
+
+    def test_yaml_dump(self):
+        """
+        Test yaml.dump a dict
+        """
+        data = {"foo": "bar"}
+        exp_yaml = "{foo: bar}\n"
+
+        assert salt.utils.yamldumper.dump(data) == exp_yaml
+
+        assert salt.utils.yamldumper.dump(
+            data, default_flow_style=False
+        ) == exp_yaml.replace("{", "").replace("}", "")
+
+    def test_yaml_safe_dump(self):
+        """
+        Test yaml.safe_dump a dict
+        """
+        data = {"foo": "bar"}
+        assert salt.utils.yamldumper.safe_dump(data) == "{foo: bar}\n"
+
+        assert (
+            salt.utils.yamldumper.safe_dump(data, default_flow_style=False)
+            == "foo: bar\n"
+        )
+
+    def test_yaml_loader_with_opts(self):
+        """
+        Test SaltYamlSafeLoader with __opts__
+        """
+        yaml_data = "key: value\n"
+        opts = {"some_key": "some_value"}
+        loader = salt.utils.yamlloader.SaltYamlSafeLoader(yaml_data, opts=opts)
+        data = loader.get_single_data()
+
+        # Check that data is parsed correctly
+        assert data == {"key": "value"}
+
+        # Check that __opts__ is accessible
+        assert loader.__opts__ == opts
+
+    def test_construct_mapping_with_opts(self):
+        """
+        Test construct_mapping with __opts__
+        """
+        yaml_data = """
+        key1: value1
+        key2: value2
+        """
+        opts = {"custom_behavior": "test_value"}
+        loader = salt.utils.yamlloader.SaltYamlSafeLoader(yaml_data, opts=opts)
+        mapping = loader.get_single_data()
+
+        # Ensure mapping was constructed correctly
+        assert mapping == {"key1": "value1", "key2": "value2"}
+
+        # Optionally, add logic to test how `__opts__` affects the loader
+        assert loader.__opts__.get("custom_behavior") == "test_value"
+
+    def test_allow_duplicate_includes(self):
+        """
+        Test allow_duplicate_includes=True
+        """
+        yaml_data = """
+        include:
+          - foo
+          - bar
+        include:
+          - foo
+          - bar
+        """
+        opts = {"allow_duplicate_includes": True}
+        loader = salt.utils.yamlloader.SaltYamlSafeLoader(yaml_data, opts=opts)
+        data = loader.get_single_data()
+
+        assert data == {
+            "include": ["foo", "bar"],
+        }
+
+    def test_allow_duplicate_includes_false(self):
+        """
+        Test allow_duplicate_includes=False
+        """
+        yaml_data = """
+        include:
+          - foo
+          - bar
+        include:
+          - foo
+          - bar
+        """
+        opts = {"allow_duplicate_includes": False}
+
+        with pytest.raises(ConstructorError, match="found conflicting ID 'include'"):
+            loader = salt.utils.yamlloader.SaltYamlSafeLoader(yaml_data, opts=opts)
+            loader.get_single_data()