Generate sha for files in reference

docs/api/scripting.md

@@ -72,3 +72,39 @@ if __name__ == '__main__':
 ```
 
 Both `document_hook` and `item_hook` are optional, but if provided these callbacks will be passed each corresponding instance. Each callback should yield instances of Doorstop's exception classes based on severity of the issue.
+
+## Validation Hook per folder
+
+Doorstop also has an extension which allows creating an item validation per folder, allowing the document to have different validations for each document section.
+To enable this mechanism you must insert into your `.doorstop.yml` file the following lines:
+
+```yaml
+extensions:
+  item_validator: .req_sha_item_validator.py # a python file path relative to .doorstop.yml
+```
+
+or
+
+```yaml
+extensions:
+  item_validator: ../validators/my_complex_validator.py # a python file path relative to .doorstop.yml
+```
+
+The referenced file must have a function called `item_validator` with a single parameter `item`.
+
+Example:
+
+```python
+
+
+def item_validator(item):
+    if getattr(item, "references") == None:
+        return [] # early return
+    for ref in item.references:
+        if ref['sha'] != item._hash_reference(ref['path']):
+            yield DoorstopError("Hash has changed and it was not reviewed properly")
+
+```
+
+Although it is not required, it is recommended to yield a Doorstop type such as,
+`DoorstopInfo`, `DoorstopError`, or `DoorstopWarning`.

docs/reference/item.md

@@ -243,10 +243,47 @@ of the item.
 
 ```yaml
 references:
-- path: tests/test1.cpp
-  type: file
-- path: tests/test2.cpp
-  type: file
+  - path: tests/test1.cpp
+    type: file
+  - path: tests/test2.cpp
+    type: file
+```
+
+### Generating hash for referenced files
+
+Doorstop has an extension to generate a hash for each referenced file inside the array.
+
+.doorstop.yml
+
+```yaml
+settings:
+  digits: 3
+  prefix: REQ
+  sep: ""
+extensions:
+  item_sha_required: true
+```
+
+Now when reviewing items, Doorstop will insert a field named `sha` where each item reference will
+contain a `sha256`.
+
+Example:
+
+```yaml
+active: true
+derived: false
+header: ""
+level: 2.0
+links: []
+normative: true
+ref: ""
+references:
+  - path: files/a.file
+    sha: 28c16553011a46bca9b78d189f8fd30c59c4138a1b6a9a4961f525849d48037e
+    type: file
+reviewed: BU95pdUUcz5DrFur8GUUqBaIXSNPBYMEZVMy-6IPM4s=
+text: |
+  My text
 ```
 
 ### Note: new behavior vs old behavior

doorstop/cli/tests/test_all.py

@@ -24,7 +24,7 @@
 from doorstop.core.tests.helpers import on_error_with_retry
 
 REQ_COUNT = 23
-ALL_COUNT = 55
+ALL_COUNT = 57
 
 
 class TempTestCase(unittest.TestCase):

doorstop/core/document.py

@@ -6,12 +6,17 @@
 import re
 from collections import OrderedDict
 from itertools import chain
-from typing import Dict, List
+from typing import Any, Dict, List
 
 import yaml
 
 from doorstop import common, settings
-from doorstop.common import DoorstopError, DoorstopInfo, DoorstopWarning
+from doorstop.common import (
+    DoorstopError,
+    DoorstopInfo,
+    DoorstopWarning,
+    import_path_as_module,
+)
 from doorstop.core.base import (
     BaseFileObject,
     BaseValidatable,
@@ -68,6 +73,7 @@ def __init__(self, path, root=os.getcwd(), **kwargs):
         self._data["parent"] = None  # type: ignore
         self._data["itemformat"] = kwargs.get("itemformat")  # type: ignore
         self._extended_reviewed: List[str] = []
+        self.extensions: Dict[str, Any] = {}
         self._items: List[Item] = []
         self._itered = False
         self.children: List[Document] = []
@@ -228,6 +234,9 @@ def load(self, reload=False):
                     key, self.config
                 )
                 raise DoorstopError(msg)
+
+        self.extensions = data.get("extensions", {})
+
         # Set meta attributes
         self._loaded = True
         if reload:
@@ -759,6 +768,11 @@ def _reorder_automatic(items, start=None, keep=None):
             # Save the current level as the previous level
             plevel = clevel.copy()
 
+    @staticmethod
+    def _import_validator(path: str):
+        """Load an external python item validator logic from doorstop yml defined for the folder."""
+        return import_path_as_module(path)
+
     @staticmethod
     def _items_by_level(items, keep=None):
         """Iterate through items by level with the kept item first."""
@@ -816,6 +830,15 @@ def get_issues(
         """
         assert document_hook is None
         skip = [] if skip is None else skip
+
+        ext_validator = None
+        if "item_validator" in self.extensions:  # type: ignore
+            validator = self.extensions["item_validator"]  # type: ignore
+            path = os.path.join(self.path, validator)  # type: ignore
+            ext_validator = self._import_validator(path)
+            ext_validator = getattr(ext_validator, "item_validator")
+
+        extension_validator = ext_validator if ext_validator else lambda **kwargs: []
         hook = item_hook if item_hook else lambda **kwargs: []
 
         if self.prefix in skip:
@@ -844,6 +867,7 @@ def get_issues(
             for issue in chain(
                 hook(item=item, document=self, tree=self.tree),
                 item_validator.get_issues(item, skip=skip),
+                extension_validator(item=item),
             ):
                 # Prepend the item's UID to yielded exceptions
                 if isinstance(issue, Exception):

doorstop/core/item.py

@@ -3,6 +3,7 @@
 """Representation of an item in a document."""
 
 import functools
+import hashlib
 import linecache
 import os
 from typing import Any, List
@@ -263,7 +264,8 @@ def _set_attributes(self, attributes):
                     if "keyword" in ref_dict:
                         ref_keyword = ref_dict["keyword"]
                         stripped_ref_dict["keyword"] = ref_keyword
-
+                    if "sha" in ref_dict:
+                        stripped_ref_dict["sha"] = ref_dict["sha"]
                     stripped_value.append(stripped_ref_dict)
 
                 value = stripped_value
@@ -297,6 +299,37 @@ def load(self, reload=False):
         # Set meta attributes
         self._loaded = True
 
+    def _hash_reference(self, path):
+        """
+        Extension method created to generate checksum from the list of find_references.
+
+        :param path: a path from a reference inside the references list
+        """
+        sha = None
+        if "item_sha_required" not in self.document.extensions:
+            return sha
+
+        try:
+            sha256 = hashlib.sha256()
+            with open(os.path.join(self.root, path), "rb") as f:
+                BUFSIZE = (
+                    int(self.document.extensions["item_sha_buffer_size"])
+                    if "item_sha_buffer_size" in self.document.extensions
+                    else 65536
+                )
+                while True:
+                    fdata = f.read(BUFSIZE)
+                    if not fdata:
+                        break
+                    sha256.update(fdata)
+                sha = sha256.hexdigest()
+
+        # We don't report missing files because validate already does that.
+        except FileNotFoundError:
+            pass
+
+        return sha
+
     @edit_item
     def save(self):
         """Format and save the item's properties to its file."""
@@ -363,6 +396,8 @@ def _yaml_data(self, textattributekeys=None):
                     if "keyword" in el:
                         ref_dict["keyword"] = el["keyword"]  # type: ignore
 
+                    if "sha" in el:
+                        ref_dict["sha"] = el["sha"]  # type: ignore
                     stripped_value.append(ref_dict)
 
                 value = stripped_value  # type: ignore
@@ -852,6 +887,30 @@ def clear(self, parents=None):
     def review(self):
         """Mark the item as reviewed."""
         log.info("marking item as reviewed...")
+
+        # only introduce the sha for reference files if
+        # the option item_sha_required is defined in the extension section from .doorstop.yml
+        if (
+            "item_sha_required" in self.document.extensions
+            and self.references is not None
+        ):
+            references = self.references
+            for ref, _ in enumerate(references):
+                temp_sha = self._hash_reference(references[ref]["path"])
+                log.info(references[ref])
+                if "sha" in references[ref] and references[ref]["sha"] == temp_sha:
+                    log.info(
+                        f"{references[ref]['path']} checksum did not change skipping update..."
+                    )
+                    continue
+
+                if "sha" in references[ref] and references[ref]["sha"] != temp_sha:
+                    log.info(f"updating checksum for {references[ref]['path']}")
+                    references[ref]["sha"] = temp_sha
+                else:
+                    log.info(f"Inserting checksum for {references[ref]['path']}")
+                    references[ref]["sha"] = temp_sha
+
         self._data["reviewed"] = self.stamp(links=True)
 
     @delete_item

doorstop/core/publishers/tests/test_publisher_html_doc.py

@@ -34,6 +34,7 @@ def setUp(self):
     traceability.csv
     traceability.html
     documents/
+        EXT.html
         HLT.html
         LLT.html
         REQ.html

doorstop/core/publishers/tests/test_publisher_latex_doc.py

@@ -37,13 +37,15 @@ def setUp(self):
         self.dirpath = os.path.abspath(os.path.join("mock_%s" % __name__, self.hex))
         os.makedirs(self.dirpath)
         self.expected_walk = """{n}/
+    EXT.tex
     HLT.tex
     LLT.tex
     REQ.tex
     Requirements.tex
     TUT.tex
     Tutorial.tex
     compile.sh
+    doc-EXT.tex
     doc-HLT.tex
     doc-LLT.tex
     traceability.tex

doorstop/core/publishers/tests/test_publisher_markdown_doc.py

@@ -29,6 +29,7 @@ def setUp(self):
         self.dirpath = os.path.abspath(os.path.join("mock_%s" % __name__, self.hex))
         os.makedirs(self.dirpath)
         self.expected_walk = """{n}/
+    EXT.md
     HLT.md
     LLT.md
     REQ.md

doorstop/core/tests/__init__.py

@@ -4,7 +4,7 @@
 
 import logging
 import os
-from typing import List
+from typing import Any, Dict, List
 from unittest.mock import MagicMock, Mock, patch
 
 from doorstop.core.base import BaseFileObject
@@ -95,6 +95,7 @@ def __init__(self):
         self.itemformat = "yaml"
         self._items: List[Item] = []
         self.extended_reviewed: List[str] = []
+        self.extensions: Dict[str, Any] = {}
 
     def __iter__(self):
         yield from self._items
@@ -103,6 +104,15 @@ def set_items(self, items):
         self._items = items
 
 
+class MockSimpleDocumentExtensions(MockSimpleDocument):
+    """Mock Document class that enable extensions."""
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        for k, v in kwargs.items():
+            self.extensions[k] = v
+
+
 class MockDocumentSkip(MockDocument):  # pylint: disable=W0223,R0902
     """Mock Document class that is always skipped in tree placement."""