Merge branch 'main' into fix-infinite-loading

open-metadata · Jan 24, 2025 · f6fd577 · f6fd577
2 parents ab0c2bc + 2099799
commit f6fd577
Show file tree

Hide file tree

Showing 92 changed files with 12,935 additions and 2,023 deletions.
diff --git a/.github/workflows/playwright-integration-tests-mysql.yml b/.github/workflows/playwright-integration-tests-mysql.yml
@@ -103,7 +103,7 @@ jobs:
           source env/bin/activate
           make install_e2e_tests
           make run_e2e_tests
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: playwright-artifacts

diff --git a/.github/workflows/playwright-integration-tests-postgres.yml b/.github/workflows/playwright-integration-tests-postgres.yml
@@ -103,7 +103,7 @@ jobs:
           source env/bin/activate
           make install_e2e_tests
           make run_e2e_tests
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: playwright-artifacts

diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml
@@ -164,21 +164,21 @@ jobs:
 
     - name: Upload coverage artifact for Python tests
       if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-${{ matrix.e2e-test }}
         path: .coverage
 
     - name: Upload coverage artifact for CLI E2E tests
-      if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
-      uses: actions/upload-artifact@v3
+        if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-${{ matrix.e2e-test }}
         path: .coverage.${{ matrix.e2e-test }}
 
     - name: Upload tests artifact
       if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: tests-${{ matrix.e2e-test }}
         path: ingestion/junit/test-results-*.xml

diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml
@@ -107,7 +107,7 @@ jobs:
 
       - name: Upload Snyk Report HTML files
         if: steps.security-report.outcome == 'success'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: security-report
           path: security-report

diff --git a/ingestion/src/metadata/clients/aws_client.py b/ingestion/src/metadata/clients/aws_client.py
@@ -58,12 +58,13 @@ class AWSAssumeRoleCredentialResponse(BaseModel):
 
 
 class AWSAssumeRoleCredentialWrapper(BaseModel):
-    accessKeyId: str = Field()
-    secretAccessKey: CustomSecretStr = Field()
-    sessionToken: Optional[str] = Field(
-        default=None,
-    )
-    expiryTime: Optional[str] = Field()
+    accessKeyId: str = Field(alias="access_key")
+    secretAccessKey: CustomSecretStr = Field(alias="secret_key")
+    sessionToken: Optional[str] = Field(default=None, alias="token")
+    expiryTime: Optional[str] = Field(alias="expiry_time")
+
+    class Config:
+        populate_by_name = True
 
 
 AWSAssumeRoleCredentialFormat = TypeVar(

diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
@@ -70,7 +70,6 @@
     get_columns,
     get_etable_owner,
     get_foreign_keys,
-    get_json_fields_and_type,
     get_table_comment,
     get_table_owner,
     get_view_definition,
@@ -113,7 +112,6 @@
 Inspector.get_all_table_ddls = get_all_table_ddls
 Inspector.get_table_ddl = get_table_ddl
 Inspector.get_table_owner = get_etable_owner
-Inspector.get_json_fields_and_type = get_json_fields_and_type
 
 PGDialect.get_foreign_keys = get_foreign_keys
 

diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py
@@ -212,91 +212,6 @@
     ORDER BY 1
 """
 
-POSTGRES_GET_JSON_FIELDS = """
-    WITH RECURSIVE json_hierarchy AS (
-        SELECT
-            key AS path,
-            json_typeof(value) AS type,
-            value,
-            json_build_object() AS properties,
-            key AS title
-        FROM
-            {table_name} tbd,
-            LATERAL json_each({column_name}::json)
-    ),
-    build_hierarchy AS (
-        SELECT
-            path,
-            type,
-            title,
-            CASE
-                WHEN type = 'object' THEN
-                    json_build_object(
-                        'title', title,
-                        'type', 'object',
-                        'properties', (
-                            SELECT json_object_agg(
-                                key,
-                                json_build_object(
-                                    'title', key,
-                                    'type', json_typeof(value),
-                                    'properties', (
-                                        CASE
-                                            WHEN json_typeof(value) = 'object' THEN
-                                                (
-                                                    SELECT json_object_agg(
-                                                        key,
-                                                        json_build_object(
-                                                            'title', key,
-                                                            'type', json_typeof(value),
-                                                            'properties',
-                                                            json_build_object()
-                                                        )
-                                                    )
-                                                    FROM json_each(value::json) AS sub_key_value
-                                                )
-                                            ELSE json_build_object()
-                                        END
-                                    )
-                                )
-                            )
-                            FROM json_each(value::json) AS key_value
-                        )
-                    )
-                WHEN type = 'array' THEN
-                    json_build_object(
-                        'title', title,
-                        'type', 'array',
-                        'properties', json_build_object()
-                    )
-                ELSE
-                    json_build_object(
-                        'title', title,
-                        'type', type
-                    )
-            END AS hierarchy
-        FROM
-            json_hierarchy
-    ),
-    aggregate_hierarchy AS (
-        select
-            json_build_object(
-            'title','{column_name}',
-            'type','object',
-            'properties',
-            json_object_agg(
-                path,
-                hierarchy
-            )) AS result
-        FROM
-            build_hierarchy
-    )
-    SELECT
-        result
-    FROM
-        aggregate_hierarchy;
-"""
-
 POSTGRES_GET_STORED_PROCEDURES = """
     SELECT proname AS procedure_name,
         nspname AS schema_name,

diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/utils.py b/ingestion/src/metadata/ingestion/source/database/postgres/utils.py
@@ -13,7 +13,6 @@
 """
 Postgres SQLAlchemy util methods
 """
-import json
 import re
 import traceback
 from typing import Dict, Optional, Tuple
@@ -24,18 +23,15 @@
 from sqlalchemy.engine import reflection
 from sqlalchemy.sql import sqltypes
 
-from metadata.generated.schema.entity.data.table import Column
 from metadata.ingestion.source.database.postgres.queries import (
     POSTGRES_COL_IDENTITY,
     POSTGRES_FETCH_FK,
-    POSTGRES_GET_JSON_FIELDS,
     POSTGRES_GET_SERVER_VERSION,
     POSTGRES_SQL_COLUMNS,
     POSTGRES_TABLE_COMMENTS,
     POSTGRES_TABLE_OWNERS,
     POSTGRES_VIEW_DEFINITIONS,
 )
-from metadata.parsers.json_schema_parser import parse_json_schema
 from metadata.utils.logger import utils_logger
 from metadata.utils.sqlalchemy_utils import (
     get_table_comment_wrapper,
@@ -190,28 +186,6 @@ def get_table_comment(
     )
 
 
-@reflection.cache
-def get_json_fields_and_type(
-    self, table_name, column_name, schema=None, **kw
-):  # pylint: disable=unused-argument
-    try:
-        query = POSTGRES_GET_JSON_FIELDS.format(
-            table_name=table_name, column_name=column_name
-        )
-        cursor = self.engine.execute(query)
-        result = cursor.fetchone()
-        if result:
-            parsed_column = parse_json_schema(json.dumps(result[0]), Column)
-            if parsed_column:
-                return parsed_column[0].children
-    except Exception as err:
-        logger.warning(
-            f"Unable to parse the json fields for {table_name}.{column_name} - {err}"
-        )
-        logger.debug(traceback.format_exc())
-    return None
-
-
 @reflection.cache
 def get_columns(  # pylint: disable=too-many-locals
     self, connection, table_name, schema=None, **kw

diff --git a/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py b/ingestion/src/metadata/ingestion/source/database/sql_column_handler.py
@@ -202,26 +202,6 @@ def _process_complex_col_type(self, parsed_string: dict, column: dict) -> Column
             ]
         return Column(**parsed_string)
 
-    @calculate_execution_time()
-    def process_json_type_column_fields(  # pylint: disable=too-many-locals
-        self, schema_name: str, table_name: str, column_name: str, inspector: Inspector
-    ) -> Optional[List[Column]]:
-        """
-        Parse fields column with json data types
-        """
-        try:
-            if hasattr(inspector, "get_json_fields_and_type"):
-                result = inspector.get_json_fields_and_type(
-                    table_name, column_name, schema_name
-                )
-                return result
-
-        except NotImplementedError:
-            logger.debug(
-                "Cannot parse json fields for table column [{schema_name}.{table_name}.{col_name}]: NotImplementedError"
-            )
-        return None
-
     @calculate_execution_time()
     def get_columns_and_constraints(  # pylint: disable=too-many-locals
         self, schema_name: str, table_name: str, db_name: str, inspector: Inspector
@@ -298,11 +278,6 @@ def get_columns_and_constraints(  # pylint: disable=too-many-locals
                     )
                     col_data_length = 1 if col_data_length is None else col_data_length
 
-                    if col_type == "JSON":
-                        children = self.process_json_type_column_fields(
-                            schema_name, table_name, column.get("name"), inspector
-                        )
-
                     om_column = Column(
                         name=ColumnName(
                             root=column["name"]

diff --git a/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md b/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md
@@ -15,6 +15,7 @@
 {% connectorInfoCard name="Doris" stage="PROD" href="/connectors/database/doris" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Druid" stage="PROD" href="/connectors/database/druid" platform="OpenMetadata" / %}
 {% connectorInfoCard name="DynamoDB" stage="PROD" href="/connectors/database/dynamodb" platform="OpenMetadata" / %}
+{% connectorInfoCard name="Exasol" stage="PROD" href="/connectors/database/exasol" platform="OpenMetadata" / %}
 {% connectorInfoCard name="GCS Datalake" stage="PROD" href="/connectors/database/gcs-datalake" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Glue" stage="PROD" href="/connectors/database/glue" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Greenplum" stage="BETA" href="/connectors/database/greenplum" platform="OpenMetadata" / %}

diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md b/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md
@@ -17,6 +17,7 @@
 {% connectorInfoCard name="Doris" stage="PROD" href="/connectors/database/doris" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Druid" stage="PROD" href="/connectors/database/druid" platform="OpenMetadata" / %}
 {% connectorInfoCard name="DynamoDB" stage="PROD" href="/connectors/database/dynamodb" platform="OpenMetadata" / %}
+{% connectorInfoCard name="Exasol" stage="PROD" href="/connectors/database/exasol" platform="OpenMetadata" / %}
 {% connectorInfoCard name="GCS Datalake" stage="PROD" href="/connectors/database/gcs-datalake" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Glue" stage="PROD" href="/connectors/database/glue" platform="OpenMetadata" / %}
 {% connectorInfoCard name="Greenplum" stage="BETA" href="/connectors/database/greenplum" platform="OpenMetadata" / %}

diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/admin-guide/alerts.md b/openmetadata-docs/content/v1.5.x/how-to-guides/admin-guide/alerts.md
@@ -5,6 +5,15 @@ slug: /how-to-guides/admin-guide/alerts
 
 # Set up Data Observability Alerts and Notifications
 
+{%inlineCalloutContainer%}
+ {%inlineCallout
+    icon="MdGppGood"
+    bold="Required Permissions"
+    href="/how-to-guides/admin-guide/roles-policies"%}
+    Setting up alerts requires create permission on the `Eventsubscription` entity. More info here.
+ {%/inlineCallout%}
+{%/inlineCalloutContainer%}
+
 OpenMetadata has been providing alerts right from the start to notify users of important data lifecycle events: schema modifications, ownership shifts, and tagging updates. Users can define fine-grained alerts and notifications.
 
 Starting from the 1.3 release, Data Observability alerts have been completely revamped, simplifying the process of monitoring data. Users can quickly create alerts for:

diff --git a/openmetadata-docs/content/v1.6.x/collate-menu.md b/openmetadata-docs/content/v1.6.x/collate-menu.md
@@ -100,6 +100,10 @@ site_menu:
     url: /connectors/database/dynamodb
   - category: Connectors / Database / DynamoDB / Run Externally
     url: /connectors/database/dynamodb/yaml
+  - category: Connectors / Database / Exasol
+    url: /connectors/database/exasol
+  - category: Connectors / Database / Exasol / Run Externally
+    url: /connectors/database/exasol/yaml
   - category: Connectors / Database / GCS Datalake
     url: /connectors/database/gcs-datalake
   - category: Connectors / Database / GCS Datalake / Run Externally