router loads init

hynky1999 · May 11, 2023 · 1f809e0 · 1f809e0
1 parent 5e3a198
commit 1f809e0
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 4 deletions.
diff --git a/cmoncrawl/processor/pipeline/extractor.py b/cmoncrawl/processor/pipeline/extractor.py
@@ -92,7 +92,11 @@ def __init__(self, filter_non_ok: bool = True):
         self.filter_non_ok = filter_non_ok
 
     def extract_soup(self, soup: BeautifulSoup, metadata: PipeMetadata):
-        metadata.name = metadata.domain_record.url.replace("/", "_")[:100]
+        metadata.name = (
+            metadata.domain_record.url.replace("/", "_")[:100]
+            if metadata.domain_record.url is not None
+            else "unknown"
+        )
         result_dict: Dict[str, Any] = {"html": str(soup)}
 
         return result_dict
@@ -120,7 +124,11 @@ def __init__(self, filter_non_ok: bool = True):
         self.filter_non_ok = filter_non_ok
 
     def extract_soup(self, soup: BeautifulSoup, metadata: PipeMetadata):
-        metadata.name = metadata.domain_record.url.replace("/", "_")[:100]
+        metadata.name = (
+            metadata.domain_record.url.replace("/", "_")[:100]
+            if metadata.domain_record.url is not None
+            else "unknown"
+        )
         result_dict: Dict[str, Any] = {
             "domain_record": metadata.domain_record.to_dict()
         }

diff --git a/cmoncrawl/processor/pipeline/router.py b/cmoncrawl/processor/pipeline/router.py
@@ -41,7 +41,7 @@ def __init__(self):
         self.registered_routes: List[Route] = []
         self.modules: Dict[str, IExtractor] = {}
 
-    def load_module(self, module_path: Path) -> IExtractor:
+    def load_module(self, module_path: Path):
         module_name = os.path.splitext(os.path.basename(module_path))[0]
         spec = importlib.util.spec_from_file_location(module_name, module_path)
         if spec is None:
@@ -53,7 +53,10 @@ def load_module(self, module_path: Path) -> IExtractor:
             raise Exception("Failed to load module: " + module_name)
 
         spec.loader.exec_module(module)
+        return module, module_name
 
+    def load_module_as_extractor(self, module_path: Path):
+        module, module_name = self.load_module(module_path)
         name: str = getattr(module, "NAME", module_name)
         extractor: IExtractor | None = getattr(module, "extractor", None)
         if extractor is None:
@@ -69,7 +72,10 @@ def load_modules(self, folder: Path):
                 if not file.endswith(".py"):
                     continue
 
-                extractors.append(self.load_module(Path(root) / file))
+                if file == "__init__.py":
+                    self.load_module(Path(root) / file)
+
+                extractors.append(self.load_module_as_extractor(Path(root) / file))
         all_purpose_logger.info(f"Loaded {len(extractors)} extractors")
 
     def load_extractor(self, name: str, extractor: IExtractor):