Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement] Support restore/rollback sync during conversion (1/2) #569

Open
wants to merge 31 commits into
base: main
Choose a base branch
from
Open

[Enhancement] Support restore/rollback sync during conversion (1/2) #569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
e0a8710
Add a source identifier in target table transaction
danielhumanmod Oct 27, 2024
9bc8256
detect rollback from source and target table
danielhumanmod Oct 27, 2024
76e0bc0
boundary safe
danielhumanmod Oct 28, 2024
45b279a
fix existing tests
danielhumanmod Oct 28, 2024
8ac5717
support commit level info
danielhumanmod Nov 22, 2024
53a58b6
remove redundant changes
danielhumanmod Nov 22, 2024
bb5ed78
simplify source id generation
danielhumanmod Nov 22, 2024
9b8b71e
remove rollback detection in this PR
danielhumanmod Nov 22, 2024
252ca2c
format
danielhumanmod Nov 22, 2024
d9cb5fd
avoid hard code
danielhumanmod Nov 22, 2024
c8dde9e
format
danielhumanmod Nov 22, 2024
bfbb78f
use timestamp as source identifier for Hudi
danielhumanmod Nov 22, 2024
002d91b
optimize early termination logic
danielhumanmod Nov 24, 2024
bf5f0a0
source-target identifier mapping test
danielhumanmod Nov 24, 2024
56fe85e
introduce source metadata
danielhumanmod Dec 4, 2024
a40cd6c
make some methods private
danielhumanmod Dec 11, 2024
1e39d1a
make xtable metadata commit level in delta and iceberg
danielhumanmod Dec 13, 2024
0af0a82
simplify hudi metadata operation
danielhumanmod Dec 13, 2024
82b2d2d
refactor get target commit process
danielhumanmod Dec 13, 2024
7ffda25
format
danielhumanmod Dec 13, 2024
4198e5b
improve doc and test
danielhumanmod Dec 14, 2024
b181a07
format
danielhumanmod Dec 14, 2024
4ca9c9a
Merge branch 'main' of https://github.com/danielhumanmod/incubator-xt…
danielhumanmod Jan 23, 2025
9552679
fix compile error
danielhumanmod Jan 23, 2025
5fe489e
make sourceIdentifier nonnull without default value
danielhumanmod Jan 25, 2025
42c804b
keep existing table-level metadata
danielhumanmod Jan 25, 2025
e470e98
standard way for doc
danielhumanmod Jan 25, 2025
2033c3a
complete tests after making sourceIdentifier nonnull
danielhumanmod Jan 25, 2025
edb99fd
using TimelineUtils#getCommitMetadata to extract commit metadata
danielhumanmod Jan 25, 2025
65dc4c9
Merge branch 'main' of https://github.com/danielhumanmod/incubator-xt…
danielhumanmod Feb 12, 2025
246ce6f
fix testing issue after merge main
danielhumanmod Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions xtable-api/src/main/java/org/apache/xtable/model/InternalSnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;

import lombok.Builder;
import lombok.NonNull;
import lombok.Value;

import org.apache.xtable.model.storage.PartitionFileGroup;
Expand All @@ -47,4 +48,6 @@ public class InternalSnapshot {
List<PartitionFileGroup> partitionedDataFiles;
// pending commits before latest commit on the table.
@Builder.Default List<Instant> pendingCommits = Collections.emptyList();
// commit identifier in source table
@NonNull String sourceIdentifier;
}
4 changes: 4 additions & 0 deletions xtable-api/src/main/java/org/apache/xtable/model/TableChange.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.xtable.model;

import lombok.Builder;
import lombok.NonNull;
import lombok.Value;

import org.apache.xtable.model.storage.DataFilesDiff;
Expand All @@ -36,4 +37,7 @@ public class TableChange {

/** The {@link InternalTable} at the commit time to which this table change belongs. */
InternalTable tableAsOfChange;

// Commit identifier in source table
@NonNull String sourceIdentifier;
}
14 changes: 12 additions & 2 deletions xtable-api/src/main/java/org/apache/xtable/model/metadata/TableSyncMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,20 @@ public class TableSyncMetadata {
Instant lastInstantSynced;
List<Instant> instantsToConsiderForNextSync;
int version;
String sourceTableFormat;
String sourceIdentifier;

public static TableSyncMetadata of(
Instant lastInstantSynced, List<Instant> instantsToConsiderForNextSync) {
return new TableSyncMetadata(lastInstantSynced, instantsToConsiderForNextSync, CURRENT_VERSION);
Instant lastInstantSynced,
List<Instant> instantsToConsiderForNextSync,
String sourceTableFormat,
String sourceIdentifier) {
return new TableSyncMetadata(
lastInstantSynced,
instantsToConsiderForNextSync,
CURRENT_VERSION,
sourceTableFormat,
sourceIdentifier);
}

public String toJson() {
Expand Down
14 changes: 14 additions & 0 deletions xtable-api/src/main/java/org/apache/xtable/spi/extractor/ConversionSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,18 @@ public interface ConversionSource<COMMIT> extends Closeable {
* false.
*/
boolean isIncrementalSyncSafeFrom(Instant instant);

/**
* Extract the identifier of the provided commit. The identifier is defined as:
*
* <ul>
* <li>Snapshot ID in Iceberg
* <li>Version ID in Delta
* <li>Timestamp in Hudi
* </ul>
*
* @param commit The provided commit
* @return the string version of the commit identifier
*/
String getCommitIdentifier(COMMIT commit);
}
10 changes: 10 additions & 0 deletions xtable-api/src/main/java/org/apache/xtable/spi/sync/ConversionTarget.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,14 @@ public interface ConversionTarget {

/** Initializes the client with provided configuration */
void init(TargetTable targetTable, Configuration configuration);

/**
* Retrieves the commit identifier from the target table that corresponds to a given source table
* commit identifier
Comment on lines +95 to +96
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will these identifier's always be the same? if so, is it simpler to make this a boolean method?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will these identifier's always be the same? if so, is it simpler to make this a boolean method?

Yes the source identifier stored in the target table remains unchanged. However, this API is designed to retrieve the corresponding target COMMIT based on an input source identifier (It would be the rollback source COMMIT in this feature). This allows us to initiate a rollback to the retrieved target COMMIT on the target table

*
* @param sourceIdentifier the unique identifier of the source table commit
* @return an {@link Optional} containing the target commit identifier if a corresponding commit
* exists, or an empty {@link Optional} if no match is found
*/
Optional<String> getTargetCommitIdentifier(String sourceIdentifier);
}
23 changes: 16 additions & 7 deletions xtable-api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public Map<String, SyncResult> syncSnapshot(
internalTable,
target -> target.syncFilesForSnapshot(snapshot.getPartitionedDataFiles()),
startTime,
snapshot.getPendingCommits()));
snapshot.getPendingCommits(),
snapshot.getSourceIdentifier()));
} catch (Exception e) {
log.error("Failed to sync snapshot", e);
results.put(
Expand Down Expand Up @@ -121,7 +122,8 @@ public Map<String, List<SyncResult>> syncChanges(
change.getTableAsOfChange(),
target -> target.syncFilesForDiff(change.getFilesDiff()),
startTime,
changes.getPendingCommits()));
changes.getPendingCommits(),
change.getSourceIdentifier()));
} catch (Exception e) {
log.error("Failed to sync table changes", e);
resultsForFormat.add(buildResultForError(SyncMode.INCREMENTAL, startTime, e));
Expand Down Expand Up @@ -149,19 +151,26 @@ private SyncResult getSyncResult(
InternalTable tableState,
SyncFiles fileSyncMethod,
Instant startTime,
List<Instant> pendingCommits) {
List<Instant> pendingCommits,
String sourceIdentifier) {
// initialize the sync
conversionTarget.beginSync(tableState);
// Persist the latest commit time in table properties for incremental syncs
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here We need to move the Metadata set operation earlier because it will be required during the sync() operation in Iceberg (Delta and Hudi only need it in completeSync()

// Syncing metadata must precede the following steps to ensure that the metadata is available
// before committing
TableSyncMetadata latestState =
TableSyncMetadata.of(
tableState.getLatestCommitTime(),
pendingCommits,
tableState.getTableFormat(),
sourceIdentifier);
conversionTarget.syncMetadata(latestState);
// sync schema updates
conversionTarget.syncSchema(tableState.getReadSchema());
// sync partition updates
conversionTarget.syncPartitionSpec(tableState.getPartitioningFields());
// Update the files in the target table
fileSyncMethod.sync(conversionTarget);
// Persist the latest commit time in table properties for incremental syncs.
TableSyncMetadata latestState =
TableSyncMetadata.of(tableState.getLatestCommitTime(), pendingCommits);
conversionTarget.syncMetadata(latestState);
conversionTarget.completeSync();

return SyncResult.builder()
Expand Down
15 changes: 9 additions & 6 deletions xtable-api/src/test/java/org/apache/xtable/model/metadata/TestTableSyncMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,17 @@ private static Stream<Arguments> provideMetadataAndJson() {
Instant.parse("2020-07-04T10:15:30.00Z"),
Arrays.asList(
Instant.parse("2020-08-21T11:15:30.00Z"),
Instant.parse("2024-01-21T12:15:30.00Z"))),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[\"2020-08-21T11:15:30Z\",\"2024-01-21T12:15:30Z\"],\"version\":0}"),
Instant.parse("2024-01-21T12:15:30.00Z")),
"TEST",
"0"),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[\"2020-08-21T11:15:30Z\",\"2024-01-21T12:15:30Z\"],\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"),
Arguments.of(
TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), Collections.emptyList()),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[],\"version\":0}"),
TableSyncMetadata.of(
Instant.parse("2020-07-04T10:15:30.00Z"), Collections.emptyList(), "TEST", "0"),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"instantsToConsiderForNextSync\":[],\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"),
Arguments.of(
TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), null),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"version\":0}"));
TableSyncMetadata.of(Instant.parse("2020-07-04T10:15:30.00Z"), null, "TEST", "0"),
"{\"lastInstantSynced\":\"2020-07-04T10:15:30Z\",\"version\":0,\"sourceTableFormat\":\"TEST\",\"sourceIdentifier\":\"0\"}"));
}

@Test
Expand Down
10 changes: 9 additions & 1 deletion xtable-api/src/test/java/org/apache/xtable/spi/extractor/TestExtractFromSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ public void extractSnapshot() {
InternalTable table = InternalTable.builder().latestCommitTime(Instant.now()).build();
List<PartitionFileGroup> dataFiles = Collections.emptyList();
InternalSnapshot internalSnapshot =
InternalSnapshot.builder().table(table).partitionedDataFiles(dataFiles).build();
InternalSnapshot.builder()
.table(table)
.partitionedDataFiles(dataFiles)
.sourceIdentifier("0")
.build();
when(mockConversionSource.getCurrentSnapshot()).thenReturn(internalSnapshot);
assertEquals(internalSnapshot, ExtractFromSource.of(mockConversionSource).extractSnapshot());
}
Expand Down Expand Up @@ -86,6 +90,7 @@ public void extractTableChanges() {
.tableAsOfChange(tableAtFirstInstant)
.filesDiff(
DataFilesDiff.builder().fileAdded(newFile1).fileRemoved(initialFile2).build())
.sourceIdentifier("0")
.build();
when(mockConversionSource.getTableChangeForCommit(firstCommitToSync))
.thenReturn(tableChangeToReturnAtFirstInstant);
Expand All @@ -94,6 +99,7 @@ public void extractTableChanges() {
.tableAsOfChange(tableAtFirstInstant)
.filesDiff(
DataFilesDiff.builder().fileAdded(newFile1).fileRemoved(initialFile2).build())
.sourceIdentifier("0")
.build();

// add 2 new files, remove 2 files
Expand All @@ -110,6 +116,7 @@ public void extractTableChanges() {
.filesAdded(Arrays.asList(newFile2, newFile3))
.filesRemoved(Arrays.asList(initialFile3, newFile1))
.build())
.sourceIdentifier("1")
.build();
when(mockConversionSource.getTableChangeForCommit(secondCommitToSync))
.thenReturn(tableChangeToReturnAtSecondInstant);
Expand All @@ -121,6 +128,7 @@ public void extractTableChanges() {
.filesAdded(Arrays.asList(newFile2, newFile3))
.filesRemoved(Arrays.asList(initialFile3, newFile1))
.build())
.sourceIdentifier("1")
.build();

IncrementalTableChanges actual =
Expand Down
Loading
Loading