Cordex extension and trying to build on higher level abstractions

CHANGES.md

@@ -20,7 +20,8 @@
 * move argument parsing for logging options to the implementation code
 * fix bug where logging options were being set incorrectly
 * rename files to avoid potential naming conflicts with other packages (`logging` and `requests`)
-
+* Add `cordex6` extension and `CORDEX-CMIP6_Ouranos` implementation. This includes a refactoring of base extension classes.
+* Add an `xscen` extension demonstrating how to add properties to a STAC Item.
 
 ## [0.6.0](https://github.com/crim-ca/stac-populator/tree/0.6.0) (2024-02-22)
 

Makefile

@@ -6,6 +6,7 @@ APP_NAME    := STACpopulator
 APP_VERSION ?= 0.6.0
 
 DOCKER_COMPOSE_FILES := -f "$(APP_ROOT)/docker/docker-compose.yml"
+COMPOSE_PROJECT_NAME := stac-populator
 DOCKER_TAG := ghcr.io/crim-ca/stac-populator:$(APP_VERSION)
 
 IMP_DIR := $(APP_NAME)/implementations
@@ -35,6 +36,7 @@ del-cmip6:
 	curl --location --request DELETE '$(STAC_HOST)/collections/CMIP6_UofT'
 	@echo ""
 
+
 docker-start:
 	docker compose $(DOCKER_COMPOSE_FILES) up
 starthost: docker-start
@@ -43,12 +45,12 @@ docker-stop:
 	docker compose $(DOCKER_COMPOSE_FILES) down
 stophost: docker-stop
 
+del_docker_volume: stophost
+	docker volume rm stac-db
+
 docker-build:
 	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile" -t "$(DOCKER_TAG)"
 
-del_docker_volume: stophost
-	docker volume rm docker_stac-db
-
 resethost: del_docker_volume starthost
 
 install:

README.md

@@ -19,13 +19,15 @@ contain all the logic for constructing the STAC representation for an item in th
 
 Provided implementations of `STACpopulatorBase`:
 
-| Implementation                 | Description                                                                                                             |
-|--------------------------------|-------------------------------------------------------------------------------------------------------------------------|
-| [CMIP6_UofT][CMIP6_UofT]       | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. |
-| [DirectoryLoader][DirLoader]   | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint.          |
+| Implementation                               | Description                                                                                                             |
+|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| [CMIP6_UofT][CMIP6_UofT]                     | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. |
+| [DirectoryLoader][DirLoader]                 | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint.          |
+| [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | 
 
 [CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py
 [DirLoader]: STACpopulator/implementations/DirectoryLoader/crawl_directory.py
+[CORDEX-CMIP6_Ouranos]: STACpopulator/implementations/CORDEX-CMIP6_Ouranos/add_CORDEX-CMIP6.py
 
 ## Installation and Execution
 

STACpopulator/cli.py

@@ -11,8 +11,7 @@
 from STACpopulator.log import setup_logging
 
 
-def add_parser_args(parser: argparse.ArgumentParser) -> dict[str, Callable]:
-    """Common CLI arguments for all implementations."""
+def add_parser_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--version",
         "-V",

STACpopulator/extensions/base.py

@@ -24,7 +24,8 @@
 """
 from __future__ import annotations
 
-from datetime import datetime
+from abc import ABC, abstractmethod
+from datetime import datetime as Datetime
 import json
 import jsonschema
 import logging
@@ -41,21 +42,21 @@
 from pystac import STACValidationError
 from pystac.extensions.base import S  # generic pystac.STACObject
 from STACpopulator.models import AnyGeometry, GeoJSONPolygon
-from STACpopulator.stac_utils import (
-    ServiceType,
-    ncattrs_to_bbox,
-    ncattrs_to_geometry,
-)
+from STACpopulator.stac_utils import ServiceType
 import types
-from STACpopulator.extensions.datacube import DataCubeHelper
-from STACpopulator.extensions.thredds import THREDDSHelper
+import uuid
+from pathlib import Path
+import tempfile
 
 T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition)
 
 LOGGER = logging.getLogger(__name__)
 
+class Helper:
+    """Class to be subclassed by extension helpers."""
+
 
-class ExtensionHelper(BaseModel):
+class ExtensionHelper(BaseModel, Helper):
     """Base class for dataset properties going into the catalog.
 
     Subclass this with attributes.
@@ -65,7 +66,7 @@ class ExtensionHelper(BaseModel):
     _prefix : str
         If not None, this prefix is added to ingested data before the jsonschema validation.
     _schema_uri : str
-        URI of the json schema to validate against.
+        URI of the json schema to validate against. Note this is not a STAC schema, but a schema for the dataset properties only.
     _schema_exclude : list[str]
         Properties not meant to be validated by json schema, but still included in the data model.
     """
@@ -75,14 +76,23 @@ class ExtensionHelper(BaseModel):
 
     model_config = ConfigDict(populate_by_name=True, extra="ignore")
 
+    @classmethod
+    def __init_subclass__(cls, **kwargs):
+        """Automatically set an alias generator from the `_prefix`."""
+        prefix = cls._prefix.default
+
+        if prefix is not None:
+            cls.model_config["alias_generator"] = lambda key: f"{prefix}:{key}"
+
     @model_validator(mode="before")
     @classmethod
     def validate_jsonschema(cls, data):
         """Validate the data model against the json schema, if given."""
         # Load schema
         uri =  cls._schema_uri.default
         if uri is not None:
-            schema = json.load(open(uri))
+            with open(uri) as f:
+                schema = json.load(f)
             validator_cls = jsonschema.validators.validator_for(schema)
             validator_cls.check_schema(schema)
             validator = validator_cls(schema)
@@ -94,15 +104,30 @@ def validate_jsonschema(cls, data):
 
         return data
 
-    def apply(self, item, add_if_missing=False):
+    def apply(self, item, add_if_missing=True):
         """Add extension for the properties of the dataset to the STAC item.
         The extension class is created dynamically from the properties.
         """
-        ExtSubCls = metacls_extension(self._prefix, schema_uri=str(self._schema_uri))
+        schema_uri = self.write_stac_schema() if self._schema_uri else None
+        ExtSubCls = metacls_extension(self._prefix, schema_uri=schema_uri)
         item_ext = ExtSubCls.ext(item, add_if_missing=add_if_missing)
         item_ext.apply(self.model_dump(mode="json", by_alias=True))
         return item
 
+    def to_stac_schema(self) -> dict:
+        """Return the STAC schema for the extension."""
+        return {'type': 'object',
+                'required': ['type', 'properties'],
+                'properties': {'type': {'const': 'Feature'},
+                               'properties': {'$ref': str(self._schema_uri)}
+                               }
+                }
+    def write_stac_schema(self) -> str:
+        path = Path(tempfile.mkdtemp()) / f"{self._prefix}-schema.json"
+        with open(path, "w") as fh:
+            json.dump(self.to_stac_schema(), fh)
+        return path
+
 
 class BaseSTAC(BaseModel):
     """Base class for STAC item data models.
@@ -117,41 +142,56 @@ class BaseSTAC(BaseModel):
         The start datetime of the dataset.
     end_datetime : datetime
         The end datetime of the dataset.
-    extensions : list[str]
-        Name of the class attributes that point to STAC extension helper classes. Those extension classes should have an `apply` method.
+    id : str
+        The unique identifier of the dataset. If not set, it should be generated by a subclass' `create_uid`.
     """
     # STAC item properties
     geometry: AnyGeometry | None
     bbox: list[float]
-    start_datetime: datetime
-    end_datetime: datetime
-
-    extensions: list = []
+    start_datetime: Datetime = None
+    end_datetime: Datetime = None
+    datetime: Datetime = None
+    id: str = None
 
     model_config = ConfigDict(populate_by_name=True, extra="ignore", arbitrary_types_allowed=True)
 
-    @property
-    def uid(self) -> str:
-        """Return a unique ID. When subclassing, use a combination of properties uniquely identifying a dataset."""
-        # TODO: Should this be an abstract method?
-        import uuid
-        return str(uuid.uuid4())
+
+    # Helpers are automatically detected by being Helper subclasses
+    _helpers: list[str] = PrivateAttr([])
+
+    @model_validator(mode="after")
+    def set_id(self):
+        """Set the ID of the dataset if None"""
+        if self.id is None:
+            self.id = self.create_uid()
+        return self
+
+    @abstractmethod
+    def create_uid(self):
+        """Return a unique identifier for the dataset."""
+
+    @model_validator(mode="after")
+    def find_helpers(self):
+        """Populate the list of extensions."""
+        for key, field in self.model_fields.items():
+            if isinstance(field.annotation, type) and issubclass(field.annotation, Helper):
+                self._helpers.append(key)
+        return self
 
     def stac_item(self) -> "pystac.Item":
         """Create a STAC item and add extensions."""
         item = pystac.Item(
-            id=self.uid,
+            id=self.id,
             geometry=self.geometry.model_dump(),
             bbox=self.bbox,
-            properties={
-                "start_datetime": str(self.start_datetime),
-                "end_datetime": str(self.end_datetime),
-            },
-            datetime=None,
+            start_datetime=self.start_datetime,
+            end_datetime=self.end_datetime,
+            datetime=self.datetime,
+            properties={},
         )
 
         # Add extensions
-        for ext in self.extensions:
+        for ext in self._helpers:
             getattr(self, ext).apply(item)
 
         try:
@@ -162,62 +202,6 @@ def stac_item(self) -> "pystac.Item":
         return json.loads(json.dumps(item.to_dict()))
 
 
-class THREDDSCatalogDataModel(BaseSTAC):
-    """Base class ingesting attributes loaded by `THREDDSLoader` and creating a STAC item.
-
-    This is meant to be subclassed for each extension.
-
-    It includes two validation mechanisms:
-     - pydantic validation using type hints, and
-     - json schema validation.
-    """
-    # Data from loader
-    data: dict
-
-    # Extensions classes
-    properties: ExtensionHelper
-    datacube: DataCubeHelper
-    thredds: THREDDSHelper
-
-    extensions: list = ["properties", "datacube", "thredds"]
-
-    model_config = ConfigDict(populate_by_name=True, extra="ignore", arbitrary_types_allowed=True)
-
-    @classmethod
-    def from_data(cls, data):
-        """Instantiate class from data provided by THREDDS Loader.
-        """
-        # This is where we match the Loader's output to the STAC item and extensions inputs. If we had multiple
-        # loaders, that's probably the only thing that would be different between them.
-        return cls(data=data,
-                   start_datetime=data["groups"]["CFMetadata"]["attributes"]["time_coverage_start"],
-                   end_datetime=data["groups"]["CFMetadata"]["attributes"]["time_coverage_end"],
-                   geometry=ncattrs_to_geometry(data),
-                   bbox=ncattrs_to_bbox(data),
-                   )
-
-    @model_validator(mode="before")
-    @classmethod
-    def properties_helper(cls, data):
-        """Instantiate the properties helper."""
-        data["properties"] = data['data']['attributes']
-        return data
-
-    @model_validator(mode="before")
-    @classmethod
-    def datacube_helper(cls, data):
-        """Instantiate the DataCubeHelper."""
-        data["datacube"] = DataCubeHelper(data['data'])
-        return data
-
-    @model_validator(mode="before")
-    @classmethod
-    def thredds_helper(cls, data):
-        """Instantiate the THREDDSHelper."""
-        data["thredds"] = THREDDSHelper(data['data']["access_urls"])
-        return data
-
-
 def metacls_extension(name, schema_uri):
     """Create an extension class dynamically from the properties."""
     cls_name = f"{name.upper()}Extension"
@@ -237,7 +221,7 @@ class MetaExtension:
     schema_uri: str
 
     def apply(self, properties: dict[str, Any]) -> None:
-        """Applies CMIP6 Extension properties to the extended
+        """Applies Extension properties to the extended
         :class:`~pystac.Item` or :class:`~pystac.Asset`.
         """
         for prop, val in properties.items():
@@ -250,9 +234,6 @@ def get_schema_uri(cls) -> str:
 
     @classmethod
     def has_extension(cls, obj: S):
-        # FIXME: this override should be removed once an official and versioned schema is released
-        # ignore the original implementation logic for a version regex
-        # since in our case, the VERSION_REGEX is not fulfilled (ie: using 'main' branch, no tag available...)
         ext_uri = cls.get_schema_uri()
         return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions)