Merge branch 'main' into main

.gitignore

@@ -190,7 +190,7 @@ tags
 # Persistent undo
 [._]*.un~
 
-.DS_Store
+*.DS_Store
 
 # Ruff cache
 .ruff_cache/

CHANGELOG.md

@@ -1,8 +1,71 @@
+## 0.16.12-dev0
+
+### Enhancements
+
+- **Prepare auto-partitioning for pluggable partitioners**. Move toward a uniform partitioner call signature so a custom or override partitioner can be registered without code changes.
+
+### Features
+
+### Fixes
+
+## 0.16.11
+
+### Enhancements
+
+- **Enhance quote standardization tests** with additional Unicode scenarios
+- **Relax table segregation rule in chunking.** Previously a `Table` element was always segregated into its own pre-chunk such that the `Table` appeared alone in a chunk or was split into multiple `TableChunk` elements, but never combined with `Text`-subtype elements. Allow table elements to be combined with other elements in the same chunk when space allows.
+- **Compute chunk length based solely on `element.text`.** Previously `.metadata.text_as_html` was also considered and since it is always longer that the text (due to HTML tag overhead) it was the effective length criterion. Remove text-as-html from the length calculation such that text-length is the sole criterion for sizing a chunk.
+
+### Features
+
+### Fixes
+
+- Fix ipv4 regex to correctly include up to three digit octets.
+
+## 0.16.10
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+- **Fix original file doctype detection** from cct converted file paths for metrics calculation.
+
+## 0.16.9
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+- **Fix NLTK Download** to not download from unstructured S3 Bucket
+
+## 0.16.8
+
+### Enhancements
+- **Metrics: Weighted table average is optional**
+
+### Features
+
+### Fixes
+
+## 0.16.7
+
+### Enhancements
+- **Add image_alt_mode to partition_html** Adds an `image_alt_mode` parameter to `partition_html()` to control how alt text is extracted from images in HTML documents for `html_parser_version=v2` . The parameter can be set to `to_text` to extract alt text as text from `<img>` html tags
+
+### Features
+
+### Fixes
+
+
 ## 0.16.6
 
 ### Enhancements
-- **Every <table> tag is considered to be ontology.Table** Added special handling for tables in HTML partitioning. This change is made to improve the accuracy of table extraction from HTML documents.
-- **Every HTML has default ontology class assigned** When parsing HTML to ontology each defined HTML in the Ontology has assigned default ontology class. This way it is possible to assign ontology class instead of UncategorizedText when the HTML tag is predicted correctly without class assigned class
+- **Every `<table>` tag is considered to be ontology.Table** Added special handling for tables in HTML partitioning (`html_parser_version=v2`. This change is made to improve the accuracy of table extraction from HTML documents.
+- **Every HTML has default ontology class assigned** When parsing HTML with `html_parser_version=v2` to ontology each defined HTML in the Ontology has assigned default ontology class. This way it is possible to assign ontology class instead of UncategorizedText when the HTML tag is predicted correctly without class assigned class
 - **Use (number of actual table) weighted average for table metrics** In evaluating table metrics the mean aggregation now uses the actual number of tables in a document to weight the metric scores
 
 ### Features

Makefile

@@ -198,6 +198,10 @@ test-extra-pypandoc:
 test-extra-xlsx:
 	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/partition/test_xlsx.py
 
+.PHONY: test-text-extraction-evaluate
+test-text-extraction-evaluate:
+	PYTHONPATH=. CI=$(CI) ${PYTHON} -m pytest test_unstructured/metrics/test_text_extraction.py
+
 ## check:                   runs linters (includes tests)
 .PHONY: check
 check: check-ruff check-black check-flake8 check-version

scripts/html/rendered_html_from_elements.py

@@ -0,0 +1,146 @@
+# pyright: reportPrivateUsage=false
+
+"""
+Script to render HTML from unstructured elements.
+NOTE: This script is not intended to be used as a module.
+NOTE: For now script is only intended to be used with elements generated with
+      `partition_html(html_parser_version=v2)`
+TODO: It was noted that unstructured_elements_to_ontology func always returns a single page
+      This script is using helper functions to handle multiple pages.
+"""
+
+import argparse
+import logging
+import os
+import select
+import sys
+from collections import defaultdict
+from typing import List, Sequence
+
+from bs4 import BeautifulSoup
+
+from unstructured.documents import elements
+from unstructured.partition.html.transformations import unstructured_elements_to_ontology
+from unstructured.staging.base import elements_from_json
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def extract_document_div(html_content: str) -> str:
+    pos = html_content.find(">")
+    if pos != -1:
+        return html_content[: pos + 1]
+    logger.error("No '>' found in the HTML content.")
+    raise ValueError("No '>' found in the HTML content.")
+
+
+def extract_page_div(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, "html.parser")
+    page_divs = soup.find_all("div", class_="Page")
+    if len(page_divs) != 1:
+        logger.error(
+            "Expected exactly one <div> element with class 'Page'. Found %d.", len(page_divs)
+        )
+        raise ValueError("Expected exactly one <div> element with class 'Page'.")
+    return str(page_divs[0])
+
+
+def fold_document_div(
+    html_document_start: str, html_document_end: str, html_per_page: List[str]
+) -> str:
+    html_document = html_document_start
+    for page_html in html_per_page:
+        html_document += page_html
+    html_document += html_document_end
+    return html_document
+
+
+def group_elements_by_page(
+    unstructured_elements: Sequence[elements.Element],
+) -> Sequence[Sequence[elements.Element]]:
+    pages_dict = defaultdict(list)
+
+    for element in unstructured_elements:
+        page_number = element.metadata.page_number
+        pages_dict[page_number].append(element)
+
+    pages_list = list(pages_dict.values())
+    return pages_list
+
+
+def rendered_html(*, filepath: str | None = None, text: str | None = None) -> str:
+    """Renders HTML from a JSON file with unstructured elements.
+
+    Args:
+        filepath (str): path to JSON file with unstructured elements.
+
+    Returns:
+        str: HTML content.
+    """
+    if filepath is None and text is None:
+        logger.error("Either filepath or text must be provided.")
+        raise ValueError("Either filepath or text must be provided.")
+    if filepath is not None and text is not None:
+        logger.error("Both filepath and text cannot be provided.")
+        raise ValueError("Both filepath and text cannot be provided.")
+    if filepath is not None:
+        logger.info("Rendering HTML from file: %s", filepath)
+    else:
+        logger.info("Rendering HTML from text.")
+
+    unstructured_elements = elements_from_json(filename=filepath, text=text)
+    unstructured_elements_per_page = group_elements_by_page(unstructured_elements)
+    # parsed_ontology = unstructured_elements_to_ontology(unstructured_elements)
+    parsed_ontology_per_page = [
+        unstructured_elements_to_ontology(elements) for elements in unstructured_elements_per_page
+    ]
+    html_per_page = [parsed_ontology.to_html() for parsed_ontology in parsed_ontology_per_page]
+
+    html_document_start = extract_document_div(html_per_page[0])
+    html_document_end = "</div>"
+    html_per_page = [extract_page_div(page) for page in html_per_page]
+
+    return fold_document_div(html_document_start, html_document_end, html_per_page)
+
+
+def _main():
+    if os.getenv("PROCESS_FROM_STDIN") == "true":
+        logger.info("Processing from STDIN (PROCESS_FROM_STDIN is set to 'true')")
+        if select.select([sys.stdin], [], [], 0.1)[0]:
+            content = sys.stdin.read()
+            html = rendered_html(text=content)
+            sys.stdout.write(html)
+        else:
+            logger.error("No input provided via STDIN. Exiting.")
+            sys.exit(1)
+    else:
+        logger.info("Processing from command line arguments")
+        parser = argparse.ArgumentParser(description="Render HTML from unstructured elements.")
+        parser.add_argument(
+            "filepath", help="Path to JSON file with unstructured elements.", type=str
+        )
+        parser.add_argument(
+            "--outdir",
+            help="Path to directory where the rendered html will be stored.",
+            type=str,
+            default=None,
+            nargs="?",
+        )
+        args = parser.parse_args()
+
+        html = rendered_html(filepath=args.filepath)
+        if args.outdir is None:
+            args.outdir = os.path.dirname(args.filepath)
+        os.makedirs(args.outdir, exist_ok=True)
+        outpath = os.path.join(
+            args.outdir, os.path.basename(args.filepath).replace(".json", ".html")
+        )
+        with open(outpath, "w") as f:
+            f.write(html)
+        logger.info("HTML rendered and saved to: %s", outpath)
+
+
+if __name__ == "__main__":
+    _main()

scripts/user/u-tables-inspect.sh

@@ -45,6 +45,8 @@ jq -c '.[] | select(.type == "Table") | .metadata.text_as_html' "$JSON_FILE" | w
   HTML_CONTENT=${HTML_CONTENT%\"}
   # add a border and padding to clearly see cell definition
   # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table /<table border="1" cellpadding="10" /')
+  # shellcheck disable=SC2001
   HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table>/<table border="1" cellpadding="10">/')
   # add newlines for readability in the html
   # shellcheck disable=SC2001