Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[590] Add Delta Glue Catalog Sync implementation #637

Merged
merged 1 commit into from
Feb 6, 2025
Merged

[590] Add Delta Glue Catalog Sync implementation #637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions xtable-aws/src/main/java/org/apache/xtable/glue/GlueCatalogTableBuilderFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.apache.xtable.catalog.CatalogTableBuilder;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.glue.table.DeltaGlueCatalogTableBuilder;
import org.apache.xtable.glue.table.IcebergGlueCatalogTableBuilder;
import org.apache.xtable.model.storage.TableFormat;

Expand All @@ -35,6 +36,8 @@ static CatalogTableBuilder<TableInput, Table> getInstance(
switch (tableFormat) {
case TableFormat.ICEBERG:
return new IcebergGlueCatalogTableBuilder(configuration);
case TableFormat.DELTA:
return new DeltaGlueCatalogTableBuilder();
default:
throw new NotSupportedException("Unsupported table format: " + tableFormat);
}
Expand Down
38 changes: 38 additions & 0 deletions xtable-aws/src/main/java/org/apache/xtable/glue/GlueSchemaExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.xtable.glue;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand All @@ -39,6 +40,7 @@

import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.SchemaExtractorException;
import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalSchema;

Expand Down Expand Up @@ -231,4 +233,40 @@ protected String toTypeString(InternalSchema fieldSchema, String tableFormat) {
protected static String getColumnProperty(String tableFormat, String property) {
return String.format("%s.%s", tableFormat.toLowerCase(Locale.ENGLISH), property);
}

public List<Column> getNonPartitionColumns(InternalTable table, Map<String, Column> columnsMap) {
List<String> partitionKeys = getPartitionKeys(table);
return columnsMap.values().stream()
.filter(c -> !partitionKeys.contains(c.name()))
.collect(Collectors.toList());
}

public List<Column> getPartitionColumns(InternalTable table, Map<String, Column> columnsMap) {
/**
* When converting delta schema to InternalSchema, generated columns are excluded: {@link
* org.apache.xtable.delta.DeltaSchemaExtractor#toInternalSchema}. In case of partition field
* being a generated column, it won't be present in columnsMap, so defaulting to string type
* until support is there
*/
return getPartitionKeys(table).stream()
.map(
pKey ->
columnsMap.getOrDefault(pKey, Column.builder().name(pKey).type("string").build()))
.collect(Collectors.toList());
}

private List<String> getPartitionKeys(InternalTable table) {
List<String> partitionKeys = new ArrayList<>();
table
.getPartitioningFields()
.forEach(
field -> {
if (!field.getPartitionFieldNames().isEmpty()) {
partitionKeys.addAll(field.getPartitionFieldNames());
} else {
partitionKeys.add(field.getSourceField().getName());
}
});
return partitionKeys;
}
}
116 changes: 116 additions & 0 deletions xtable-aws/src/main/java/org/apache/xtable/glue/table/DeltaGlueCatalogTableBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.xtable.glue.table;

import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP;
import static org.apache.xtable.catalog.CatalogUtils.toHierarchicalTableIdentifier;
import static org.apache.xtable.catalog.Constants.PROP_EXTERNAL;
import static org.apache.xtable.catalog.Constants.PROP_PATH;
import static org.apache.xtable.catalog.Constants.PROP_SERIALIZATION_FORMAT;
import static org.apache.xtable.catalog.Constants.PROP_SPARK_SQL_SOURCES_PROVIDER;
import static org.apache.xtable.glue.GlueCatalogSyncClient.GLUE_EXTERNAL_TABLE_TYPE;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;

import com.google.common.annotations.VisibleForTesting;

import org.apache.xtable.catalog.CatalogTableBuilder;
import org.apache.xtable.glue.GlueSchemaExtractor;
import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.catalog.CatalogTableIdentifier;
import org.apache.xtable.model.catalog.HierarchicalTableIdentifier;
import org.apache.xtable.model.storage.TableFormat;

import software.amazon.awssdk.services.glue.model.Column;
import software.amazon.awssdk.services.glue.model.SerDeInfo;
import software.amazon.awssdk.services.glue.model.StorageDescriptor;
import software.amazon.awssdk.services.glue.model.Table;
import software.amazon.awssdk.services.glue.model.TableInput;

/** Delta specific table operations for Glue catalog sync */
public class DeltaGlueCatalogTableBuilder implements CatalogTableBuilder<TableInput, Table> {

private final GlueSchemaExtractor schemaExtractor;
private static final String tableFormat = TableFormat.DELTA;

public DeltaGlueCatalogTableBuilder() {
this.schemaExtractor = GlueSchemaExtractor.getInstance();
}

@Override
public TableInput getCreateTableRequest(
InternalTable table, CatalogTableIdentifier tblIdentifier) {
HierarchicalTableIdentifier tableIdentifier = toHierarchicalTableIdentifier(tblIdentifier);
Map<String, Column> columnsMap =
schemaExtractor.toColumns(tableFormat, table.getReadSchema()).stream()
.collect(Collectors.toMap(Column::name, c -> c));

return TableInput.builder()
.name(tableIdentifier.getTableName())
.tableType(GLUE_EXTERNAL_TABLE_TYPE)
.parameters(getTableParameters())
.storageDescriptor(
StorageDescriptor.builder()
.columns(schemaExtractor.getNonPartitionColumns(table, columnsMap))
.location(table.getBasePath())
.serdeInfo(SerDeInfo.builder().parameters(getSerDeParameters(table)).build())
.build())
.partitionKeys(schemaExtractor.getPartitionColumns(table, columnsMap))
.build();
}

@Override
public TableInput getUpdateTableRequest(
InternalTable table, Table catalogTable, CatalogTableIdentifier tblIdentifier) {
HierarchicalTableIdentifier tableIdentifier = toHierarchicalTableIdentifier(tblIdentifier);
Map<String, String> parameters = new HashMap<>(catalogTable.parameters());
Map<String, Column> columnsMap =
schemaExtractor.toColumns(tableFormat, table.getReadSchema(), catalogTable).stream()
.collect(Collectors.toMap(Column::name, c -> c));
return TableInput.builder()
.name(tableIdentifier.getTableName())
.tableType(GLUE_EXTERNAL_TABLE_TYPE)
.parameters(parameters)
.storageDescriptor(
catalogTable.storageDescriptor().toBuilder()
.columns(schemaExtractor.getNonPartitionColumns(table, columnsMap))
.build())
.partitionKeys(schemaExtractor.getPartitionColumns(table, columnsMap))
.build();
}

@VisibleForTesting
Map<String, String> getTableParameters() {
Map<String, String> parameters = new HashMap<>();
parameters.put(TABLE_TYPE_PROP, tableFormat);
parameters.put(PROP_SPARK_SQL_SOURCES_PROVIDER, tableFormat);
parameters.put(PROP_EXTERNAL, "TRUE");
return parameters;
}

@VisibleForTesting
Map<String, String> getSerDeParameters(InternalTable table) {
Map<String, String> parameters = new HashMap<>();
parameters.put(PROP_SERIALIZATION_FORMAT, "1");
parameters.put(PROP_PATH, table.getBasePath());
return parameters;
}
}
110 changes: 92 additions & 18 deletions xtable-aws/src/test/java/org/apache/xtable/glue/GlueCatalogSyncTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,28 @@

package org.apache.xtable.glue;

import static org.apache.xtable.glue.GlueCatalogSyncClient.GLUE_EXTERNAL_TABLE_TYPE;
import static org.apache.xtable.glue.TestGlueSchemaExtractor.getColumn;

import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.mockito.Mock;

import org.apache.xtable.conversion.ExternalCatalogConfig;
import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.catalog.ThreePartHierarchicalTableIdentifier;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.CatalogType;
import org.apache.xtable.model.storage.TableFormat;

import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.Column;
import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;
import software.amazon.awssdk.services.glue.model.CreateTableRequest;
import software.amazon.awssdk.services.glue.model.DatabaseInput;
Expand All @@ -60,17 +66,91 @@ public class GlueCatalogSyncTestBase {
protected static final String TEST_CATALOG_NAME = "aws-glue-1";
protected static final String ICEBERG_METADATA_FILE_LOCATION = "base-path/metadata";
protected static final String ICEBERG_METADATA_FILE_LOCATION_v2 = "base-path/v2-metadata";
protected static final InternalPartitionField PARTITION_FIELD =
InternalPartitionField.builder()
.sourceField(
InternalField.builder()
.name("partitionField")
.schema(
InternalSchema.builder().name("string").dataType(InternalType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
protected static final InternalSchema INTERNAL_SCHEMA =
InternalSchema.builder()
.dataType(InternalType.RECORD)
.fields(
Arrays.asList(
getInternalField("intField", "int", InternalType.INT),
getInternalField("stringField", "string", InternalType.STRING),
getInternalField("partitionField", "string", InternalType.STRING)))
.build();
protected static final InternalSchema UPDATED_INTERNAL_SCHEMA =
InternalSchema.builder()
.dataType(InternalType.RECORD)
.fields(
Arrays.asList(
getInternalField("intField", "int", InternalType.INT),
getInternalField("stringField", "string", InternalType.STRING),
getInternalField("partitionField", "string", InternalType.STRING),
getInternalField("booleanField", "boolean", InternalType.BOOLEAN)))
.build();
protected static final List<Column> PARTITION_KEYS =
Collections.singletonList(getColumn(TableFormat.DELTA, "partitionField", "string"));
protected static final List<Column> DELTA_GLUE_SCHEMA =
Arrays.asList(
getColumn(TableFormat.DELTA, "intField", "int"),
getColumn(TableFormat.DELTA, "stringField", "string"));
protected static final List<Column> UPDATED_DELTA_GLUE_SCHEMA =
Arrays.asList(
getColumn(TableFormat.DELTA, "booleanField", "boolean"),
getColumn(TableFormat.DELTA, "intField", "int"),
getColumn(TableFormat.DELTA, "stringField", "string"));
protected static final List<Column> ICEBERG_GLUE_SCHEMA =
Arrays.asList(
getColumn(TableFormat.ICEBERG, "intField", "int"),
getColumn(TableFormat.ICEBERG, "stringField", "string"),
getColumn(TableFormat.ICEBERG, "partitionField", "string"));
protected static final List<Column> UPDATED_ICEBERG_GLUE_SCHEMA =
Arrays.asList(
getColumn(TableFormat.ICEBERG, "intField", "int"),
getColumn(TableFormat.ICEBERG, "stringField", "string"),
getColumn(TableFormat.ICEBERG, "partitionField", "string"),
getColumn(TableFormat.ICEBERG, "booleanField", "boolean"));
protected static final InternalTable TEST_ICEBERG_INTERNAL_TABLE =
InternalTable.builder()
.basePath(TEST_BASE_PATH)
.tableFormat(TableFormat.ICEBERG)
.readSchema(InternalSchema.builder().fields(Collections.emptyList()).build())
.readSchema(INTERNAL_SCHEMA)
.partitioningFields(Collections.singletonList(PARTITION_FIELD))
.build();
protected static final InternalTable TEST_UPDATED_ICEBERG_INTERNAL_TABLE =
InternalTable.builder()
.basePath(TEST_BASE_PATH)
.tableFormat(TableFormat.ICEBERG)
.readSchema(UPDATED_INTERNAL_SCHEMA)
.partitioningFields(Collections.singletonList(PARTITION_FIELD))
.build();
protected static final InternalTable TEST_HUDI_INTERNAL_TABLE =
InternalTable.builder()
.basePath(TEST_BASE_PATH)
.tableFormat(TableFormat.HUDI)
.readSchema(InternalSchema.builder().fields(Collections.emptyList()).build())
.readSchema(INTERNAL_SCHEMA)
.partitioningFields(Collections.singletonList(PARTITION_FIELD))
.build();
protected static final InternalTable TEST_DELTA_INTERNAL_TABLE =
InternalTable.builder()
.basePath(TEST_BASE_PATH)
.tableFormat(TableFormat.DELTA)
.readSchema(INTERNAL_SCHEMA)
.partitioningFields(Collections.singletonList(PARTITION_FIELD))
.build();
protected static final InternalTable TEST_UPDATED_DELTA_INTERNAL_TABLE =
InternalTable.builder()
.basePath(TEST_BASE_PATH)
.tableFormat(TableFormat.DELTA)
.readSchema(UPDATED_INTERNAL_SCHEMA)
.partitioningFields(Collections.singletonList(PARTITION_FIELD))
.build();
protected static final ThreePartHierarchicalTableIdentifier TEST_CATALOG_TABLE_IDENTIFIER =
new ThreePartHierarchicalTableIdentifier(TEST_GLUE_DATABASE, TEST_GLUE_TABLE);
Expand Down Expand Up @@ -108,20 +188,6 @@ protected CreateDatabaseRequest createDbRequest(String dbName) {
.build();
}

protected TableInput getCreateOrUpdateTableInput(
String tableName, Map<String, String> params, InternalTable internalTable) {
return TableInput.builder()
.name(tableName)
.tableType(GLUE_EXTERNAL_TABLE_TYPE)
.parameters(params)
.storageDescriptor(
StorageDescriptor.builder()
.location(internalTable.getBasePath())
.columns(Collections.emptyList())
.build())
.build();
}

protected CreateTableRequest createTableRequest(String dbName, TableInput tableInput) {
return CreateTableRequest.builder()
.catalogId(TEST_GLUE_CATALOG_ID)
Expand Down Expand Up @@ -154,4 +220,12 @@ protected Table getGlueTable(String dbName, String tableName, String location) {
.storageDescriptor(StorageDescriptor.builder().location(location).build())
.build();
}

private static InternalField getInternalField(
String fieldName, String schemaName, InternalType dataType) {
return InternalField.builder()
.name(fieldName)
.schema(InternalSchema.builder().name(schemaName).dataType(dataType).build())
.build();
}
}
Loading