added supportsIncrementalMetadataExtraction in connection config

open-metadata · Jan 23, 2025 · 379885b · 379885b
1 parent b1b70cc
commit 379885b
Show file tree

Hide file tree

Showing 11 changed files with 65 additions and 64 deletions.
diff --git a/...-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json b/...-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json
@@ -25,6 +25,11 @@
       "type": "boolean",
       "default": true
     },
+    "supportsIncrementalMetadataExtraction": {
+      "description": "Supports Incremental Metadata Extraction.",
+      "type": "boolean",
+      "default": true
+    },
     "supportsUsageExtraction": {
       "description": "Supports Usage Extraction.",
       "type": "boolean",

diff --git a/...c/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json b/...c/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
@@ -80,6 +80,10 @@
       "title": "Supports Metadata Extraction",
       "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
     },
+    "supportsIncrementalMetadataExtraction": {
+      "title": "Supports Incremental Metadata Extraction",
+      "$ref": "../connectionBasicType.json#/definitions/supportsIncrementalMetadataExtraction"
+    },
     "supportsUsageExtraction": {
       "$ref": "../connectionBasicType.json#/definitions/supportsUsageExtraction"
     },

diff --git a/...c/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json b/...c/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json
@@ -81,6 +81,10 @@
       "title": "Supports Metadata Extraction",
       "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
     },
+    "supportsIncrementalMetadataExtraction": {
+      "title": "Supports Incremental Metadata Extraction",
+      "$ref": "../connectionBasicType.json#/definitions/supportsIncrementalMetadataExtraction"
+    },
     "supportsUsageExtraction": {
       "$ref": "../connectionBasicType.json#/definitions/supportsUsageExtraction"
     },

diff --git a/.../main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/.../main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
@@ -104,6 +104,10 @@
       "title": "Supports Metadata Extraction",
       "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
     },
+    "supportsIncrementalMetadataExtraction": {
+      "title": "Supports Incremental Metadata Extraction",
+      "$ref": "../connectionBasicType.json#/definitions/supportsIncrementalMetadataExtraction"
+    },
     "supportsUsageExtraction": {
       "$ref": "../connectionBasicType.json#/definitions/supportsUsageExtraction"
     },

diff --git a/...ain/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx b/...ain/resources/ui/src/components/Settings/Services/AddIngestion/AddIngestion.component.tsx
@@ -289,7 +289,7 @@ const AddIngestion = ({
             operationType={status}
             pipeLineType={pipelineType}
             serviceCategory={serviceCategory}
-            serviceType={serviceData.serviceType}
+            serviceData={serviceData}
             workflowData={workflowData}
             onCancel={handleCancelClick}
             onChange={handleDataChange}

diff --git a/...rc/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx b/...rc/components/Settings/Services/Ingestion/IngestionWorkflowForm/IngestionWorkflowForm.tsx
@@ -19,7 +19,6 @@ import { isUndefined, omit, omitBy } from 'lodash';
 import React, { FC, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import {
-  DATABASES_WITHOUT_INCREMENTAL_EXTRACTION_SUPPORT,
   EXCLUDE_INCREMENTAL_EXTRACTION_SUPPORT_UI_SCHEMA,
   INGESTION_ELASTIC_SEARCH_WORKFLOW_UI_SCHEMA,
   INGESTION_WORKFLOW_UI_SCHEMA,
@@ -28,7 +27,6 @@ import {
   DbtConfigType,
   PipelineType,
 } from '../../../../../generated/api/services/ingestionPipelines/createIngestionPipeline';
-import { DatabaseServiceType } from '../../../../../generated/entity/data/database';
 import {
   IngestionWorkflowData,
   IngestionWorkflowFormProps,
@@ -53,7 +51,7 @@ const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
   onFocus,
   onSubmit,
   onChange,
-  serviceType,
+  serviceData,
 }) => {
   const [internalData, setInternalData] =
     useState<IngestionWorkflowData>(workflowData);
@@ -75,10 +73,8 @@ const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
 
   const isDbtPipeline = pipeLineType === PipelineType.Dbt;
 
-  const isIncrementalExtractionNotSupported =
-    DATABASES_WITHOUT_INCREMENTAL_EXTRACTION_SUPPORT.includes(
-      serviceType as DatabaseServiceType
-    );
+  const isIncrementalExtractionSupported =
+    serviceData?.connection.config.supportsIncrementalMetadataExtraction;
 
   const uiSchema = useMemo(() => {
     let commonSchema = { ...INGESTION_WORKFLOW_UI_SCHEMA };
@@ -89,7 +85,7 @@ const IngestionWorkflowForm: FC<IngestionWorkflowFormProps> = ({
       };
     }
 
-    if (isIncrementalExtractionNotSupported) {
+    if (!isIncrementalExtractionSupported) {
       commonSchema = {
         ...commonSchema,
         ...EXCLUDE_INCREMENTAL_EXTRACTION_SUPPORT_UI_SCHEMA,

diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
@@ -307,12 +307,6 @@ export const INGESTION_WORKFLOW_UI_SCHEMA = {
   ],
 };
 
-export const DATABASES_WITHOUT_INCREMENTAL_EXTRACTION_SUPPORT = [
-  DatabaseServiceType.Mysql,
-  DatabaseServiceType.Postgres,
-  DatabaseServiceType.Mssql,
-];
-
 export const EXCLUDE_INCREMENTAL_EXTRACTION_SUPPORT_UI_SCHEMA = {
   incremental: {
     'ui:widget': 'hidden',

diff --git a/...ain/resources/ui/src/generated/entity/services/connections/database/bigQueryConnection.ts b/...ain/resources/ui/src/generated/entity/services/connections/database/bigQueryConnection.ts
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2024 Collate.
+ *  Copyright 2025 Collate.
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
@@ -10,9 +10,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-
-
- /**
+/**
  * Google BigQuery Connection Config
  */
 export interface BigQueryConnection {
@@ -30,16 +28,17 @@ export interface BigQueryConnection {
     /**
      * SQLAlchemy driver scheme options.
      */
-    scheme?:                     BigqueryScheme;
-    supportsDatabase?:           boolean;
-    supportsDataDiff?:           boolean;
-    supportsDBTExtraction?:      boolean;
-    supportsLineageExtraction?:  boolean;
-    supportsMetadataExtraction?: boolean;
-    supportsProfiler?:           boolean;
-    supportsQueryComment?:       boolean;
-    supportsSystemProfile?:      boolean;
-    supportsUsageExtraction?:    boolean;
+    scheme?:                                BigqueryScheme;
+    supportsDatabase?:                      boolean;
+    supportsDataDiff?:                      boolean;
+    supportsDBTExtraction?:                 boolean;
+    supportsIncrementalMetadataExtraction?: boolean;
+    supportsLineageExtraction?:             boolean;
+    supportsMetadataExtraction?:            boolean;
+    supportsProfiler?:                      boolean;
+    supportsQueryComment?:                  boolean;
+    supportsSystemProfile?:                 boolean;
+    supportsUsageExtraction?:               boolean;
     /**
      * Taxonomy location used to fetch policy tags
      */

diff --git a/...ain/resources/ui/src/generated/entity/services/connections/database/redshiftConnection.ts b/...ain/resources/ui/src/generated/entity/services/connections/database/redshiftConnection.ts
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2024 Collate.
+ *  Copyright 2025 Collate.
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
@@ -10,9 +10,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-
-
- /**
+/**
  * Redshift  Connection Config
  */
 export interface RedshiftConnection {
@@ -40,18 +38,19 @@ export interface RedshiftConnection {
     /**
      * SQLAlchemy driver scheme options.
      */
-    scheme?:                     RedshiftScheme;
-    sslConfig?:                  Config;
-    sslMode?:                    SSLMode;
-    supportsDatabase?:           boolean;
-    supportsDataDiff?:           boolean;
-    supportsDBTExtraction?:      boolean;
-    supportsLineageExtraction?:  boolean;
-    supportsMetadataExtraction?: boolean;
-    supportsProfiler?:           boolean;
-    supportsQueryComment?:       boolean;
-    supportsSystemProfile?:      boolean;
-    supportsUsageExtraction?:    boolean;
+    scheme?:                                RedshiftScheme;
+    sslConfig?:                             Config;
+    sslMode?:                               SSLMode;
+    supportsDatabase?:                      boolean;
+    supportsDataDiff?:                      boolean;
+    supportsDBTExtraction?:                 boolean;
+    supportsIncrementalMetadataExtraction?: boolean;
+    supportsLineageExtraction?:             boolean;
+    supportsMetadataExtraction?:            boolean;
+    supportsProfiler?:                      boolean;
+    supportsQueryComment?:                  boolean;
+    supportsSystemProfile?:                 boolean;
+    supportsUsageExtraction?:               boolean;
     /**
      * Service Type
      */

diff --git a/...in/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts b/...in/resources/ui/src/generated/entity/services/connections/database/snowflakeConnection.ts
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2024 Collate.
+ *  Copyright 2025 Collate.
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
@@ -10,9 +10,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-
-
- /**
+/**
  * Snowflake Connection Config
  */
 export interface SnowflakeConnection {
@@ -64,16 +62,17 @@ export interface SnowflakeConnection {
     /**
      * Snowflake Passphrase Key used with Private Key
      */
-    snowflakePrivatekeyPassphrase?: string;
-    supportsDatabase?:              boolean;
-    supportsDataDiff?:              boolean;
-    supportsDBTExtraction?:         boolean;
-    supportsLineageExtraction?:     boolean;
-    supportsMetadataExtraction?:    boolean;
-    supportsProfiler?:              boolean;
-    supportsQueryComment?:          boolean;
-    supportsSystemProfile?:         boolean;
-    supportsUsageExtraction?:       boolean;
+    snowflakePrivatekeyPassphrase?:         string;
+    supportsDatabase?:                      boolean;
+    supportsDataDiff?:                      boolean;
+    supportsDBTExtraction?:                 boolean;
+    supportsIncrementalMetadataExtraction?: boolean;
+    supportsLineageExtraction?:             boolean;
+    supportsMetadataExtraction?:            boolean;
+    supportsProfiler?:                      boolean;
+    supportsQueryComment?:                  boolean;
+    supportsSystemProfile?:                 boolean;
+    supportsUsageExtraction?:               boolean;
     /**
      * Service Type
      */

diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/service.interface.ts
@@ -22,10 +22,7 @@ import {
   DashboardConnection,
   DashboardService,
 } from '../generated/entity/services/dashboardService';
-import {
-  DatabaseService,
-  DatabaseServiceType,
-} from '../generated/entity/services/databaseService';
+import { DatabaseService } from '../generated/entity/services/databaseService';
 import {
   MessagingConnection,
   MessagingService,
@@ -138,5 +135,5 @@ export interface IngestionWorkflowFormProps {
   onFocus: (fieldId: string) => void;
   onSubmit: (data: IngestionWorkflowData) => void;
   onChange?: (data: IngestionWorkflowData) => void;
-  serviceType?: DatabaseServiceType;
+  serviceData?: any;
 }