Bump mass version to 4.0.0 and use hexkit v4 (GSI-1338) (#34)

* Bump mass version to 4.0.0 and use hexkit v4

* Ruff formatting

.devcontainer/.dev_config.yaml

@@ -1,4 +1,4 @@
-db_connection_str: mongodb://localhost:27017
+mongo_dsn: mongodb://localhost:27017
 db_name: metadata-store
 searchable_classes:
   Dataset:

.pre-commit-config.yaml

@@ -48,13 +48,13 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.3
+    rev: v0.9.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.13.0
+    rev: v1.15.0
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.pyproject_generation/pyproject_custom.toml

@@ -1,12 +1,12 @@
 [project]
 name = "mass"
-version = "3.0.4"
+version = "4.0.0"
 description = "Metadata Artifact Search Service - A service for searching metadata artifacts and filtering results."
 dependencies = [
     "typer>=0.12",
-    "ghga-service-commons[api]>=3.1.5",
+    "ghga-service-commons[api]>=3.3.0",
     "ghga-event-schemas>=3.1.1",
-    "hexkit[mongodb,akafka]>=3.5.0",
+    "hexkit[mongodb,akafka]>=4.0.0",
 ]
 
 [project.urls]

.pyproject_generation/pyproject_template.toml

@@ -40,6 +40,7 @@ fixable = [
     "UP", # e.g. List -> list
     "I", # sort imports
     "D", # pydocstyle
+    "RUF022", # sort items in __all__
 ]
 ignore = [
     "E111", # indentation with invalid multiple (for formatter)
@@ -59,6 +60,7 @@ ignore = [
     "D206", # indent-with-spaces (for formatter)
     "D300", # triple-single-quotes (for formatter)
     "UP040", # type statement (not yet supported by mypy)
+    "PLC0206", # Extracting value from dictionary without calling `.items()`
 ]
 select = [
     "C90", # McCabe Complexity

README.md

@@ -37,21 +37,21 @@ We recommend using the provided Docker container.
 
 A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/mass):
 ```bash
-docker pull ghga/mass:3.0.4
+docker pull ghga/mass:4.0.0
 ```
 
 Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile):
 ```bash
 # Execute in the repo's root dir:
-docker build -t ghga/mass:3.0.4 .
+docker build -t ghga/mass:4.0.0 .
 ```
 
 For production-ready deployment, we recommend using Kubernetes, however,
 for simple use cases, you could execute the service using docker
 on a single server:
 ```bash
 # The entrypoint is preconfigured:
-docker run -p 8080:8080 ghga/mass:3.0.4 --help
+docker run -p 8080:8080 ghga/mass:4.0.0 --help
 ```
 
 If you prefer not to use containers, you may install the service from source:
@@ -193,7 +193,92 @@ The service requires the following configuration parameters:
   ```
 
 
-- **`db_connection_str`** *(string, format: password, required)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
+- **`kafka_max_retries`** *(integer)*: The maximum number of times to immediately retry consuming an event upon failure. Works independently of the dead letter queue. Minimum: `0`. Default: `0`.
+
+
+  Examples:
+
+  ```json
+  0
+  ```
+
+
+  ```json
+  1
+  ```
+
+
+  ```json
+  2
+  ```
+
+
+  ```json
+  3
+  ```
+
+
+  ```json
+  5
+  ```
+
+
+- **`kafka_enable_dlq`** *(boolean)*: A flag to toggle the dead letter queue. If set to False, the service will crash upon exhausting retries instead of publishing events to the DLQ. If set to True, the service will publish events to the DLQ topic after exhausting all retries. Default: `false`.
+
+
+  Examples:
+
+  ```json
+  true
+  ```
+
+
+  ```json
+  false
+  ```
+
+
+- **`kafka_dlq_topic`** *(string)*: The name of the topic used to resolve error-causing events. Default: `"dlq"`.
+
+
+  Examples:
+
+  ```json
+  "dlq"
+  ```
+
+
+- **`kafka_retry_backoff`** *(integer)*: The number of seconds to wait before retrying a failed event. The backoff time is doubled for each retry attempt. Minimum: `0`. Default: `0`.
+
+
+  Examples:
+
+  ```json
+  0
+  ```
+
+
+  ```json
+  1
+  ```
+
+
+  ```json
+  2
+  ```
+
+
+  ```json
+  3
+  ```
+
+
+  ```json
+  5
+  ```
+
+
+- **`mongo_dsn`** *(string, format: multi-host-uri, required)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
 
 
   Examples:
@@ -213,6 +298,32 @@ The service requires the following configuration parameters:
   ```
 
 
+- **`mongo_timeout`**: Timeout in seconds for API calls to MongoDB. The timeout applies to all steps needed to complete the operation, including server selection, connection checkout, serialization, and server-side execution. When the timeout expires, PyMongo raises a timeout exception. If set to None, the operation will not time out (default MongoDB behavior). Default: `null`.
+
+  - **Any of**
+
+    - *integer*: Exclusive minimum: `0`.
+
+    - *null*
+
+
+  Examples:
+
+  ```json
+  300
+  ```
+
+
+  ```json
+  600
+  ```
+
+
+  ```json
+  null
+  ```
+
+
 - **`host`** *(string)*: IP of the host. Default: `"127.0.0.1"`.
 
 - **`port`** *(integer)*: Port to expose the server on the specified host. Default: `8080`.

config_schema.json

@@ -210,15 +210,62 @@
       "title": "Kafka Max Message Size",
       "type": "integer"
     },
-    "db_connection_str": {
+    "kafka_max_retries": {
+      "default": 0,
+      "description": "The maximum number of times to immediately retry consuming an event upon failure. Works independently of the dead letter queue.",
+      "examples": [
+        0,
+        1,
+        2,
+        3,
+        5
+      ],
+      "minimum": 0,
+      "title": "Kafka Max Retries",
+      "type": "integer"
+    },
+    "kafka_enable_dlq": {
+      "default": false,
+      "description": "A flag to toggle the dead letter queue. If set to False, the service will crash upon exhausting retries instead of publishing events to the DLQ. If set to True, the service will publish events to the DLQ topic after exhausting all retries",
+      "examples": [
+        true,
+        false
+      ],
+      "title": "Kafka Enable DLQ",
+      "type": "boolean"
+    },
+    "kafka_dlq_topic": {
+      "default": "dlq",
+      "description": "The name of the topic used to resolve error-causing events.",
+      "examples": [
+        "dlq"
+      ],
+      "title": "Kafka DLQ Topic",
+      "type": "string"
+    },
+    "kafka_retry_backoff": {
+      "default": 0,
+      "description": "The number of seconds to wait before retrying a failed event. The backoff time is doubled for each retry attempt.",
+      "examples": [
+        0,
+        1,
+        2,
+        3,
+        5
+      ],
+      "minimum": 0,
+      "title": "Kafka Retry Backoff",
+      "type": "integer"
+    },
+    "mongo_dsn": {
       "description": "MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/",
       "examples": [
         "mongodb://localhost:27017"
       ],
-      "format": "password",
-      "title": "Db Connection Str",
-      "type": "string",
-      "writeOnly": true
+      "format": "multi-host-uri",
+      "minLength": 1,
+      "title": "Mongo Dsn",
+      "type": "string"
     },
     "db_name": {
       "description": "Name of the database located on the MongoDB server.",
@@ -228,6 +275,25 @@
       "title": "Db Name",
       "type": "string"
     },
+    "mongo_timeout": {
+      "anyOf": [
+        {
+          "exclusiveMinimum": 0,
+          "type": "integer"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "Timeout in seconds for API calls to MongoDB. The timeout applies to all steps needed to complete the operation, including server selection, connection checkout, serialization, and server-side execution. When the timeout expires, PyMongo raises a timeout exception. If set to None, the operation will not time out (default MongoDB behavior).",
+      "examples": [
+        300,
+        600,
+        null
+      ],
+      "title": "Mongo Timeout"
+    },
     "host": {
       "default": "127.0.0.1",
       "description": "IP of the host.",
@@ -359,7 +425,7 @@
     "resource_deletion_event_type",
     "resource_upsertion_event_type",
     "kafka_servers",
-    "db_connection_str",
+    "mongo_dsn",
     "db_name"
   ],
   "title": "ModSettings",

example_config.yaml

@@ -4,12 +4,15 @@ cors_allow_credentials: null
 cors_allowed_headers: null
 cors_allowed_methods: null
 cors_allowed_origins: null
-db_connection_str: '**********'
 db_name: metadata-store
 docs_url: /docs
 generate_correlation_id: true
 host: 127.0.0.1
+kafka_dlq_topic: dlq
+kafka_enable_dlq: false
 kafka_max_message_size: 1048576
+kafka_max_retries: 0
+kafka_retry_backoff: 0
 kafka_security_protocol: PLAINTEXT
 kafka_servers:
 - kafka:9092
@@ -20,6 +23,8 @@ kafka_ssl_password: ''
 log_format: null
 log_level: INFO
 log_traceback: true
+mongo_dsn: '**********'
+mongo_timeout: null
 openapi_url: /openapi.json
 port: 8080
 resource_change_event_topic: searchable_resources