Skip to content

Commit

Permalink
fix: only add user selected streams to the catalog diff (#53)
Browse files Browse the repository at this point in the history
Co-authored-by: Ben Church <[email protected]>
  • Loading branch information
flutra-osmani and bnchrch authored Dec 8, 2023
1 parent d219b9c commit 5360c07
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ public static Set<StreamTransform> getCatalogDiff(final AirbyteCatalog oldCatalo
&& s.getStream().getName().equals(descriptor.getName()))
.findFirst();

if (!streamOld.equals(streamNew)) {
if (!streamOld.equals(streamNew) && stream.isPresent()) {
// getStreamDiff only checks for differences in the stream's field name or field type
// but there are a number of reasons the streams might be different (such as a source-defined
// primary key or cursor changing). These should not be expressed as "stream updates".
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -278,13 +278,8 @@ void testCatalogDiffWithoutStreamConfig() throws IOException {

final Set<StreamTransform> diff = CatalogHelpers.getCatalogDiff(catalog1, catalog2, configuredAirbyteCatalog);

final List<StreamTransform> expectedDiff = Stream.of(
StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName(USERS), new UpdateStreamTransform(Set.of(
FieldTransform.createRemoveFieldTransform(List.of(DATE), schema1.get(PROPERTIES).get(DATE), false),
FieldTransform.createRemoveFieldTransform(List.of("id"), schema1.get(PROPERTIES).get("id"), false)))))
.toList();

Assertions.assertThat(diff).containsAll(expectedDiff);
//configuredCatalog is for a different stream, so no diff should be found
Assertions.assertThat(diff).hasSize(0);
}

@Test
Expand Down

0 comments on commit 5360c07

Please sign in to comment.