From a23afa1ab2cfb68cc7674f565ce2d700308154c0 Mon Sep 17 00:00:00 2001 From: Jesse Jia Date: Fri, 13 Oct 2023 09:54:47 -0700 Subject: [PATCH 1/5] Add rest endpoint to support backfill relationship --- .../linkedin/metadata/dao/BaseLocalDAO.java | 4 +- .../metadata/dao/BaseLocalDAOTest.java | 6 ++- .../metadata/dao/EbeanLocalAccess.java | 8 +++- .../linkedin/metadata/dao/EbeanLocalDAO.java | 11 ++++-- .../dao/EbeanLocalRelationshipWriterDAO.java | 2 +- .../metadata/dao/IEbeanLocalAccess.java | 6 ++- .../dao/utils/EmbeddedMariaInstance.java | 10 ++--- .../metadata/restli/BaseEntityResource.java | 37 +++++++++++++++++++ .../metadata/restli/RestliConstants.java | 1 + .../metadata/restli/BackfillResult.pdl | 2 +- 10 files changed, 69 insertions(+), 18 deletions(-) diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java index bced43d4e..1549d5759 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java @@ -11,6 +11,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.UnionTemplate; import com.linkedin.metadata.backfill.BackfillMode; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; import com.linkedin.metadata.dao.equality.DefaultEqualityTester; import com.linkedin.metadata.dao.equality.EqualityTester; import com.linkedin.metadata.dao.exception.ModelValidationException; @@ -807,7 +808,8 @@ public abstract void updateLocalIndex(@Nonnull U * @param urn the URN for the entity the aspect (which the local relationship is derived from) is attached to * @param aspectClass class of the aspect to backfill */ - public abstract void backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass); + public abstract List.LocalRelationshipUpdates> + backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass); /** * Returns list of urns from local secondary index that satisfy the given filter conditions. diff --git a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java index c41688b03..b9d8f0d6a 100644 --- a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java +++ b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java @@ -2,6 +2,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; import com.linkedin.metadata.dao.producer.BaseMetadataEventProducer; import com.linkedin.metadata.dao.producer.BaseTrackingMetadataEventProducer; import com.linkedin.metadata.dao.retention.TimeBasedRetention; @@ -84,8 +85,9 @@ public void updateEntityTables(@Nonnull FooUrn u } @Override - public void backfillLocalRelationshipsFromEntityTables(@Nonnull FooUrn urn, @Nonnull Class aspectClass) { - + public List.LocalRelationshipUpdates> + backfillLocalRelationshipsFromEntityTables(@Nonnull FooUrn urn, @Nonnull Class aspectClass) { + return null; } @Nonnull diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java index 3e9bdc040..11f943f81 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java @@ -29,6 +29,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -134,13 +135,18 @@ public int add(@Nonnull URN urn, @Nullable ASPEC } @Override - public void addRelationships(@Nonnull URN urn, @Nonnull ASPECT aspect, @Nonnull Class aspectClass) { + public List.LocalRelationshipUpdates> + addRelationships(@Nonnull URN urn, @Nonnull ASPECT aspect, @Nonnull Class aspectClass) { if (_localRelationshipBuilderRegistry != null && _localRelationshipBuilderRegistry.isRegistered(aspectClass)) { List.LocalRelationshipUpdates> localRelationshipUpdates = _localRelationshipBuilderRegistry.getLocalRelationshipBuilder(aspect).buildRelationships(urn, aspect); _localRelationshipWriterDAO.processLocalRelationshipUpdates(localRelationshipUpdates); + + return localRelationshipUpdates; } + + return new ArrayList<>(); } /** diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java index dc01a2c98..d78ea11b4 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java @@ -7,6 +7,7 @@ import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.UnionTemplate; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; import com.linkedin.metadata.dao.builder.LocalRelationshipBuilderRegistry; import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.exception.RetryLimitReached; @@ -578,19 +579,21 @@ public void updateEntityTables(@Nonnull URN urn, }, 1); } - public void backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass) { + public List.LocalRelationshipUpdates> + backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass) { if (_schemaConfig == SchemaConfig.OLD_SCHEMA_ONLY) { throw new UnsupportedOperationException("Local relationship tables cannot be used in OLD_SCHEMA_ONLY mode, so they cannot be backfilled."); } AspectKey key = new AspectKey<>(aspectClass, urn, LATEST_VERSION); - runInTransactionWithRetry(() -> { + return runInTransactionWithRetry(() -> { List results = _localAccess.batchGetUnion(Collections.singletonList(key), 1, 0); if (results.size() == 0) { return null; // unused } Optional aspect = toRecordTemplate(aspectClass, results.get(0)); - aspect.ifPresent(value -> _localAccess.addRelationships(urn, value, aspectClass)); - return null; // unused + + // unused + return aspect.map(value -> _localAccess.addRelationships(urn, value, aspectClass)).orElse(null); }, 1); } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java index 47eabc3f8..2847afd80 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java @@ -95,7 +95,7 @@ private void addRelationshipGroup(@Nonnull RELATIONSHIP firstRelationship = relationshipGroup.get(0); RelationshipValidator.validateRelationshipSchema(firstRelationship.getClass()); - // Process remove option to delete some local relationships if nedded before adding new relationships. + // Process remove option to delete some local relationships if needed before adding new relationships. processRemovalOption(SQLSchemaUtils.getRelationshipTableName(firstRelationship), firstRelationship, removalOption); long now = Instant.now().toEpochMilli(); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java index b27847787..a0a6a6eb5 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java @@ -3,6 +3,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; import com.linkedin.metadata.dao.builder.LocalRelationshipBuilderRegistry; import com.linkedin.metadata.dao.scsi.UrnPathExtractor; import com.linkedin.metadata.query.IndexFilter; @@ -37,8 +38,11 @@ int add(@Nonnull URN urn, @Nullable ASPECT newVa * @param urn urn associated with the relationships * @param relationship aspect from which the relationships are derived from * @param aspectClass class of the aspect + * @return relationship updates applied on relationship table */ - void addRelationships(@Nonnull URN urn, @Nonnull ASPECT relationship, @Nonnull Class aspectClass); + @Nonnull + List.LocalRelationshipUpdates> + addRelationships(@Nonnull URN urn, @Nonnull ASPECT relationship, @Nonnull Class aspectClass); /** * Get read aspects from entity table. This a new schema implementation for batchGetUnion() in {@link EbeanLocalDAO} diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java index c37528272..88b2761e8 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java @@ -64,13 +64,9 @@ private static void initDB() { configurationBuilder.setDataDir(baseDbDir + File.separator + "data"); configurationBuilder.setBaseDir(baseDbDir + File.separator + "base"); - /* - * Add below 3 lines of code if building datahub-gma on a M1 / M2 chip Apple computer. - * - * configurationBuilder.setBaseDir("/opt/homebrew"); - * configurationBuilder.setUnpackingFromClasspath(false); - * configurationBuilder.setLibDir(System.getProperty("java.io.tmpdir") + "/MariaDB4j/no-libs"); - */ + configurationBuilder.setBaseDir("/opt/homebrew"); + configurationBuilder.setUnpackingFromClasspath(false); + configurationBuilder.setLibDir(System.getProperty("java.io.tmpdir") + "/MariaDB4j/no-libs"); try { // ensure the DB directory is deleted before we start to have a clean start diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java index 43ede702c..a5c9fc776 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java @@ -36,6 +36,7 @@ import com.linkedin.restli.server.annotations.QueryParam; import com.linkedin.restli.server.annotations.RestMethod; import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; +import java.lang.reflect.InvocationTargetException; import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; @@ -356,6 +357,42 @@ public Task backfillEntityTables(@ActionParam(PARAM_URNS) @Nonnu }); } + /** + * Backfill the relationship tables from entity table. + */ + @Action(name = ACTION_BACKFILL_RELATIONSHIP_TABLES) + @Nonnull + public Task backfillRelationshipTables(@ActionParam(PARAM_URNS) @Nonnull String[] urns, + @ActionParam(PARAM_ASPECTS) @Nonnull String[] aspectNames) { + final BackfillResult backfillResult = new BackfillResult() + .setEntities(new BackfillResultEntityArray()) + .setRelationships(new BackfillResultRelationshipArray()); + + for (String urn : urns) { + for (Class aspect : parseAspectsParam(aspectNames)) { + getLocalDAO().backfillLocalRelationshipsFromEntityTables(parseUrnParam(urn), aspect).forEach(relationshipUpdates -> { + relationshipUpdates.getRelationships().forEach(relationship -> { + try { + Urn source = (Urn) relationship.getClass().getDeclaredMethod("getSource").invoke(null); + Urn dest = (Urn) relationship.getClass().getDeclaredMethod("getDestination").invoke(null); + BackfillResultRelationship backfillResultRelationship = new BackfillResultRelationship() + .setSource(source) + .setDestination(dest) + .setRemovalOption(relationshipUpdates.getRemovalOption().name()) + .setRelationship(relationship.getClass().getCanonicalName()); + + backfillResult.getRelationships().add(backfillResultRelationship); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + }); + } + } + + return RestliUtils.toTask(() -> backfillResult); + } + /** * An action method for emitting MAE backfill messages for a set of entities using SCSI. */ diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java index aa7e5d9ee..7e15ae1e7 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/RestliConstants.java @@ -10,6 +10,7 @@ private RestliConstants() { } public static final String ACTION_AUTOCOMPLETE = "autocomplete"; public static final String ACTION_BACKFILL = "backfill"; public static final String ACTION_BACKFILL_ENTITY_TABLES = "backfillEntityTables"; + public static final String ACTION_BACKFILL_RELATIONSHIP_TABLES = "backfillRelationshipTables"; public static final String ACTION_BACKFILL_WITH_URNS = "backfillWithUrns"; public static final String ACTION_BACKFILL_WITH_NEW_VALUE = "backfillWithNewValue"; public static final String ACTION_BACKFILL_LEGACY = "backfillLegacy"; diff --git a/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl b/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl index 3fe7cc339..433059337 100644 --- a/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl +++ b/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl @@ -19,7 +19,7 @@ record BackfillResult { * List of the aspects backfilled for the entity */ aspects: array[string] - }] + }], /** * List of relationships backfilled From adb6a6454b505e700d8cf557d7e02db0f65b74dd Mon Sep 17 00:00:00 2001 From: Jesse Jia Date: Sat, 14 Oct 2023 16:41:17 -0700 Subject: [PATCH 2/5] add unit test --- .../linkedin/metadata/dao/BaseLocalDAO.java | 6 ++--- .../builder/BaseLocalRelationshipBuilder.java | 4 +-- .../metadata/dao/BaseLocalDAOTest.java | 6 ++--- .../metadata/dao/EbeanLocalAccess.java | 6 ++--- .../linkedin/metadata/dao/EbeanLocalDAO.java | 10 +++---- .../dao/EbeanLocalRelationshipWriterDAO.java | 6 ++--- .../metadata/dao/IEbeanLocalAccess.java | 6 ++--- .../metadata/dao/EbeanLocalDAOTest.java | 27 +++++++++++++++++-- .../EbeanLocalRelationshipWriterDAOTest.java | 10 +++---- .../ReportsToLocalRelationshipBuilder.java | 4 +-- .../metadata/restli/BaseEntityResource.java | 6 ++--- .../metadata/restli/BackfillResult.pdl | 2 +- .../restli/BaseEntityResourceTest.java | 27 +++++++++++++++++++ 13 files changed, 84 insertions(+), 36 deletions(-) diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java index 1549d5759..f184ac54b 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java @@ -11,7 +11,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.UnionTemplate; import com.linkedin.metadata.backfill.BackfillMode; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.equality.DefaultEqualityTester; import com.linkedin.metadata.dao.equality.EqualityTester; import com.linkedin.metadata.dao.exception.ModelValidationException; @@ -808,8 +808,8 @@ public abstract void updateLocalIndex(@Nonnull U * @param urn the URN for the entity the aspect (which the local relationship is derived from) is attached to * @param aspectClass class of the aspect to backfill */ - public abstract List.LocalRelationshipUpdates> - backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass); + public abstract List backfillLocalRelationshipsFromEntityTables( + @Nonnull URN urn, @Nonnull Class aspectClass); /** * Returns list of urns from local secondary index that satisfy the given filter conditions. diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/builder/BaseLocalRelationshipBuilder.java b/dao-api/src/main/java/com/linkedin/metadata/dao/builder/BaseLocalRelationshipBuilder.java index 9ea156b91..a4444453e 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/builder/BaseLocalRelationshipBuilder.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/builder/BaseLocalRelationshipBuilder.java @@ -13,10 +13,10 @@ */ public abstract class BaseLocalRelationshipBuilder { - private Class _aspectClass; + private final Class _aspectClass; @Value - public class LocalRelationshipUpdates { + public static class LocalRelationshipUpdates { List relationships; BaseGraphWriterDAO.RemovalOption removalOption; } diff --git a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java index b9d8f0d6a..2bf5230ff 100644 --- a/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java +++ b/dao-api/src/test/java/com/linkedin/metadata/dao/BaseLocalDAOTest.java @@ -2,7 +2,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.producer.BaseMetadataEventProducer; import com.linkedin.metadata.dao.producer.BaseTrackingMetadataEventProducer; import com.linkedin.metadata.dao.retention.TimeBasedRetention; @@ -85,8 +85,8 @@ public void updateEntityTables(@Nonnull FooUrn u } @Override - public List.LocalRelationshipUpdates> - backfillLocalRelationshipsFromEntityTables(@Nonnull FooUrn urn, @Nonnull Class aspectClass) { + public List backfillLocalRelationshipsFromEntityTables( + @Nonnull FooUrn urn, @Nonnull Class aspectClass) { return null; } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java index 11f943f81..7927862f4 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalAccess.java @@ -5,7 +5,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.SetMode; import com.linkedin.metadata.aspect.AuditedAspect; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.builder.LocalRelationshipBuilderRegistry; import com.linkedin.metadata.dao.scsi.EmptyPathExtractor; import com.linkedin.metadata.dao.scsi.UrnPathExtractor; @@ -135,10 +135,10 @@ public int add(@Nonnull URN urn, @Nullable ASPEC } @Override - public List.LocalRelationshipUpdates> + public List addRelationships(@Nonnull URN urn, @Nonnull ASPECT aspect, @Nonnull Class aspectClass) { if (_localRelationshipBuilderRegistry != null && _localRelationshipBuilderRegistry.isRegistered(aspectClass)) { - List.LocalRelationshipUpdates> localRelationshipUpdates = + List localRelationshipUpdates = _localRelationshipBuilderRegistry.getLocalRelationshipBuilder(aspect).buildRelationships(urn, aspect); _localRelationshipWriterDAO.processLocalRelationshipUpdates(localRelationshipUpdates); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java index d78ea11b4..195b6f54e 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java @@ -7,7 +7,7 @@ import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.UnionTemplate; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.builder.LocalRelationshipBuilderRegistry; import com.linkedin.metadata.dao.exception.ModelConversionException; import com.linkedin.metadata.dao.exception.RetryLimitReached; @@ -579,7 +579,7 @@ public void updateEntityTables(@Nonnull URN urn, }, 1); } - public List.LocalRelationshipUpdates> + public List backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass) { if (_schemaConfig == SchemaConfig.OLD_SCHEMA_ONLY) { throw new UnsupportedOperationException("Local relationship tables cannot be used in OLD_SCHEMA_ONLY mode, so they cannot be backfilled."); @@ -588,12 +588,10 @@ public void updateEntityTables(@Nonnull URN urn, return runInTransactionWithRetry(() -> { List results = _localAccess.batchGetUnion(Collections.singletonList(key), 1, 0); if (results.size() == 0) { - return null; // unused + return new ArrayList<>(); } Optional aspect = toRecordTemplate(aspectClass, results.get(0)); - - // unused - return aspect.map(value -> _localAccess.addRelationships(urn, value, aspectClass)).orElse(null); + return aspect.map(value -> _localAccess.addRelationships(urn, value, aspectClass)).orElse(new ArrayList<>()); }, 1); } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java index 2847afd80..45fb3cb25 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipWriterDAO.java @@ -2,7 +2,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.internal.BaseGraphWriterDAO; import com.linkedin.metadata.dao.utils.GraphUtils; import com.linkedin.metadata.dao.utils.RecordUtils; @@ -48,9 +48,9 @@ public EbeanLocalRelationshipWriterDAO(EbeanServer server) { */ @Transactional public void processLocalRelationshipUpdates( - @Nonnull List.LocalRelationshipUpdates> relationshipUpdates) { + @Nonnull List relationshipUpdates) { - for (BaseLocalRelationshipBuilder.LocalRelationshipUpdates relationshipUpdate : relationshipUpdates) { + for (LocalRelationshipUpdates relationshipUpdate : relationshipUpdates) { addRelationships(relationshipUpdate.getRelationships(), relationshipUpdate.getRemovalOption()); } } diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java index a0a6a6eb5..690617bfe 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/IEbeanLocalAccess.java @@ -3,7 +3,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.metadata.dao.builder.LocalRelationshipBuilderRegistry; import com.linkedin.metadata.dao.scsi.UrnPathExtractor; import com.linkedin.metadata.query.IndexFilter; @@ -41,8 +41,8 @@ int add(@Nonnull URN urn, @Nullable ASPECT newVa * @return relationship updates applied on relationship table */ @Nonnull - List.LocalRelationshipUpdates> - addRelationships(@Nonnull URN urn, @Nonnull ASPECT relationship, @Nonnull Class aspectClass); + List addRelationships(@Nonnull URN urn, + @Nonnull ASPECT relationship, @Nonnull Class aspectClass); /** * Get read aspects from entity table. This a new schema implementation for batchGetUnion() in {@link EbeanLocalDAO} diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java index 0e1b7a627..01e440c73 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/EbeanLocalDAOTest.java @@ -13,6 +13,7 @@ import com.linkedin.metadata.dao.EbeanLocalDAO.FindMethodology; import com.linkedin.metadata.dao.EbeanLocalDAO.SchemaConfig; import com.linkedin.metadata.dao.EbeanMetadataAspect.PrimaryKey; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; import com.linkedin.metadata.dao.equality.AlwaysFalseEqualityTester; import com.linkedin.metadata.dao.equality.DefaultEqualityTester; import com.linkedin.metadata.dao.exception.InvalidMetadataType; @@ -104,12 +105,14 @@ import org.testng.annotations.Test; import static com.linkedin.common.AuditStamps.*; +import static com.linkedin.metadata.dao.internal.BaseGraphWriterDAO.RemovalOption.*; import static com.linkedin.metadata.dao.utils.EBeanDAOUtils.*; import static com.linkedin.metadata.dao.utils.SQLSchemaUtils.*; import static com.linkedin.testing.TestUtils.*; import static org.mockito.Mockito.*; import static org.testng.Assert.*; + public class EbeanLocalDAOTest { private long _now; private EbeanServer _server; @@ -3061,16 +3064,36 @@ public void testBackfillLocalRelationshipsFromEntityTables() throws URISyntaxExc BarUrn barUrn1 = BarUrn.createFromString("urn:li:bar:1"); BarUrn barUrn2 = BarUrn.createFromString("urn:li:bar:2"); BarUrn barUrn3 = BarUrn.createFromString("urn:li:bar:3"); - AspectFooBar aspectFooBar = new AspectFooBar().setBars(new BarUrnArray(barUrn1, barUrn2, barUrn3)); + BarUrnArray barUrns = new BarUrnArray(barUrn1, barUrn2, barUrn3); + AspectFooBar aspectFooBar = new AspectFooBar().setBars(barUrns); dao.add(fooUrn, aspectFooBar, _dummyAuditStamp); // clear local relationship table _server.createSqlUpdate("delete from metadata_relationship_belongsto").execute(); + List relationshipUpdates = dao.backfillLocalRelationshipsFromEntityTables(fooUrn, AspectFooBar.class); List results = _server.createSqlQuery("select * from metadata_relationship_belongsto").findList(); - assertEquals(3, results.size()); + assertEquals(results.size(), 3); + assertEquals(relationshipUpdates.size(), 1); + assertEquals(relationshipUpdates.get(0).getRemovalOption(), REMOVE_ALL_EDGES_TO_DESTINATION); + + BarUrnArray sources = new BarUrnArray(); + for (int i = 0; i < results.size(); i++) { + try { + RecordTemplate relationship = relationshipUpdates.get(0).getRelationships().get(i); + Urn source = (Urn) relationship.getClass().getMethod("getSource").invoke(relationship); + Urn dest = (Urn) relationship.getClass().getMethod("getDestination").invoke(relationship); + assertEquals(dest.toString(), "urn:li:foo:1"); + sources.add(BarUrn.createFromString(source.toString())); + assertEquals(relationshipUpdates.get(0).getRelationships().get(i).getClass().getSimpleName(), "BelongsTo"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + assertEquals(sources, barUrns); } } diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java index 1dbda204d..b6324cdb9 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipWriterDAOTest.java @@ -7,7 +7,7 @@ import com.linkedin.metadata.dao.localrelationship.builder.ReportsToLocalRelationshipBuilder; import com.linkedin.metadata.dao.localrelationship.builder.VersionOfLocalRelationshipBuilder; import com.linkedin.metadata.dao.utils.EmbeddedMariaInstance; -import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; import com.linkedin.testing.BarUrnArray; import com.linkedin.testing.localrelationship.AspectFooBar; import com.linkedin.testing.urn.BarUrn; @@ -47,7 +47,7 @@ public void testAddRelationshipWithRemoveAllEdgesToDestination() throws URISynta BarUrn.createFromString("urn:li:bar:456"), BarUrn.createFromString("urn:li:bar:789"))); - List.LocalRelationshipUpdates> updates = new BelongsToLocalRelationshipBuilder(AspectFooBar.class) + List updates = new BelongsToLocalRelationshipBuilder(AspectFooBar.class) .buildRelationships(FooUrn.createFromString("urn:li:foo:123"), aspectFooBar); // Before processing @@ -82,7 +82,7 @@ public void testAddRelationshipWithRemoveNone() throws URISyntaxException { BarUrn.createFromString("urn:li:bar:456"), BarUrn.createFromString("urn:li:bar:789"))); - List.LocalRelationshipUpdates> updates = new ReportsToLocalRelationshipBuilder(AspectFooBar.class) + List updates = new ReportsToLocalRelationshipBuilder(AspectFooBar.class) .buildRelationships(FooUrn.createFromString("urn:li:foo:123"), aspectFooBar); // Before processing @@ -114,7 +114,7 @@ public void testAddRelationshipWithRemoveAllEdgesFromSourceToDestination() throw AspectFooBar aspectFooBar = new AspectFooBar().setBars(new BarUrnArray(BarUrn.createFromString("urn:li:bar:123"))); - List.LocalRelationshipUpdates> updates = new PairsWithLocalRelationshipBuilder(AspectFooBar.class) + List updates = new PairsWithLocalRelationshipBuilder(AspectFooBar.class) .buildRelationships(FooUrn.createFromString("urn:li:foo:123"), aspectFooBar); // Before processing @@ -151,7 +151,7 @@ public void testAddRelationshipWithRemoveAllEdgesFromSource() throws URISyntaxEx AspectFooBar aspectFooBar = new AspectFooBar().setBars(new BarUrnArray(BarUrn.createFromString("urn:li:bar:123"))); - List.LocalRelationshipUpdates> updates = new VersionOfLocalRelationshipBuilder(AspectFooBar.class) + List updates = new VersionOfLocalRelationshipBuilder(AspectFooBar.class) .buildRelationships(FooUrn.createFromString("urn:li:foo:123"), aspectFooBar); // Before processing diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/builder/ReportsToLocalRelationshipBuilder.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/builder/ReportsToLocalRelationshipBuilder.java index 8e6db8485..5d3c154f1 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/builder/ReportsToLocalRelationshipBuilder.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/builder/ReportsToLocalRelationshipBuilder.java @@ -25,8 +25,8 @@ public List buildRelationships(@Nonn reportsToRelationships.add(new ReportsTo().setSource(barUrn).setDestination(urn)); } - LocalRelationshipUpdates localRelationshipUpdates = new LocalRelationshipUpdates(reportsToRelationships, - BaseGraphWriterDAO.RemovalOption.REMOVE_NONE); + LocalRelationshipUpdates localRelationshipUpdates = + new LocalRelationshipUpdates(reportsToRelationships, BaseGraphWriterDAO.RemovalOption.REMOVE_NONE); return Collections.singletonList(localRelationshipUpdates); } diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java index a5c9fc776..f8c1ff770 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java @@ -373,13 +373,13 @@ public Task backfillRelationshipTables(@ActionParam(PARAM_URNS) getLocalDAO().backfillLocalRelationshipsFromEntityTables(parseUrnParam(urn), aspect).forEach(relationshipUpdates -> { relationshipUpdates.getRelationships().forEach(relationship -> { try { - Urn source = (Urn) relationship.getClass().getDeclaredMethod("getSource").invoke(null); - Urn dest = (Urn) relationship.getClass().getDeclaredMethod("getDestination").invoke(null); + Urn source = (Urn) relationship.getClass().getMethod("getSource").invoke(relationship); + Urn dest = (Urn) relationship.getClass().getMethod("getDestination").invoke(relationship); BackfillResultRelationship backfillResultRelationship = new BackfillResultRelationship() .setSource(source) .setDestination(dest) .setRemovalOption(relationshipUpdates.getRemovalOption().name()) - .setRelationship(relationship.getClass().getCanonicalName()); + .setRelationship(relationship.getClass().getSimpleName()); backfillResult.getRelationships().add(backfillResultRelationship); } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { diff --git a/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl b/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl index 433059337..3fe7cc339 100644 --- a/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl +++ b/restli-resources/src/main/pegasus/com/linkedin/metadata/restli/BackfillResult.pdl @@ -19,7 +19,7 @@ record BackfillResult { * List of the aspects backfilled for the entity */ aspects: array[string] - }], + }] /** * List of relationships backfilled diff --git a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java index d1415123f..368bd8821 100644 --- a/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java +++ b/restli-resources/src/test/java/com/linkedin/metadata/restli/BaseEntityResourceTest.java @@ -10,6 +10,8 @@ import com.linkedin.metadata.dao.BaseLocalDAO; import com.linkedin.metadata.dao.ListResult; import com.linkedin.metadata.dao.UrnAspectEntry; +import com.linkedin.metadata.dao.builder.BaseLocalRelationshipBuilder.LocalRelationshipUpdates; +import com.linkedin.metadata.dao.internal.BaseGraphWriterDAO; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.events.IngestionTrackingContext; @@ -39,6 +41,7 @@ import com.linkedin.testing.EntitySnapshot; import com.linkedin.testing.EntityValue; import com.linkedin.testing.localrelationship.AspectFooBar; +import com.linkedin.testing.localrelationship.BelongsTo; import com.linkedin.testing.urn.BarUrn; import com.linkedin.testing.urn.FooUrn; import java.net.URISyntaxException; @@ -681,6 +684,30 @@ public void testBackfillWithNewValue() { assertTrue(backfillResultEntity.getAspects().contains("com.linkedin.testing.AspectBar")); } + @Test + public void testBackfillRelationshipTables() { + FooUrn fooUrn = makeFooUrn(1); + BarUrn barUrn = makeBarUrn(1); + + String[] aspects = new String[]{"com.linkedin.testing.AspectFoo"}; + BelongsTo belongsTo = new BelongsTo().setSource(fooUrn).setDestination(barUrn); + List belongsTos = Collections.singletonList(belongsTo); + + LocalRelationshipUpdates updates = new LocalRelationshipUpdates(belongsTos, + BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE); + List relationships = Collections.singletonList(updates); + + when(_mockLocalDAO.backfillLocalRelationshipsFromEntityTables(fooUrn, AspectFoo.class)).thenReturn(relationships); + BackfillResult backfillResult = runAndWait(_resource.backfillRelationshipTables(new String[]{fooUrn.toString()}, aspects)); + + assertTrue(backfillResult.hasRelationships()); + assertEquals(backfillResult.getRelationships().size(), 1); + assertEquals(backfillResult.getRelationships().get(0).getDestination().toString(), "urn:li:bar:1"); + assertEquals(backfillResult.getRelationships().get(0).getSource().toString(), "urn:li:foo:1"); + assertEquals(backfillResult.getRelationships().get(0).getRelationship(), "BelongsTo"); + assertEquals(backfillResult.getRelationships().get(0).getRemovalOption(), "REMOVE_ALL_EDGES_FROM_SOURCE"); + } + @Test public void testListUrnsFromIndex() { // case 1: indexFilter is non-null From 649376afdbdf4c25d5f60cd0c055c48bf21c28c0 Mon Sep 17 00:00:00 2001 From: Jesse Jia Date: Sat, 14 Oct 2023 16:44:48 -0700 Subject: [PATCH 3/5] revert --- .../metadata/dao/utils/EmbeddedMariaInstance.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java index 88b2761e8..c37528272 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EmbeddedMariaInstance.java @@ -64,9 +64,13 @@ private static void initDB() { configurationBuilder.setDataDir(baseDbDir + File.separator + "data"); configurationBuilder.setBaseDir(baseDbDir + File.separator + "base"); - configurationBuilder.setBaseDir("/opt/homebrew"); - configurationBuilder.setUnpackingFromClasspath(false); - configurationBuilder.setLibDir(System.getProperty("java.io.tmpdir") + "/MariaDB4j/no-libs"); + /* + * Add below 3 lines of code if building datahub-gma on a M1 / M2 chip Apple computer. + * + * configurationBuilder.setBaseDir("/opt/homebrew"); + * configurationBuilder.setUnpackingFromClasspath(false); + * configurationBuilder.setLibDir(System.getProperty("java.io.tmpdir") + "/MariaDB4j/no-libs"); + */ try { // ensure the DB directory is deleted before we start to have a clean start From 469e099b89d78c83635edfeee4c580bd758fd4fd Mon Sep 17 00:00:00 2001 From: Jesse Jia Date: Mon, 16 Oct 2023 13:56:15 -0700 Subject: [PATCH 4/5] Address comment --- .../src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java | 1 + .../main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java | 4 ++-- .../java/com/linkedin/metadata/restli/BaseEntityResource.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java index f184ac54b..7bc3c236b 100644 --- a/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java +++ b/dao-api/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java @@ -807,6 +807,7 @@ public abstract void updateLocalIndex(@Nonnull U * * @param urn the URN for the entity the aspect (which the local relationship is derived from) is attached to * @param aspectClass class of the aspect to backfill + * @return A list of local relationship updates executed. */ public abstract List backfillLocalRelationshipsFromEntityTables( @Nonnull URN urn, @Nonnull Class aspectClass); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java index 195b6f54e..ef801d559 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java @@ -579,8 +579,8 @@ public void updateEntityTables(@Nonnull URN urn, }, 1); } - public List - backfillLocalRelationshipsFromEntityTables(@Nonnull URN urn, @Nonnull Class aspectClass) { + public List backfillLocalRelationshipsFromEntityTables( + @Nonnull URN urn, @Nonnull Class aspectClass) { if (_schemaConfig == SchemaConfig.OLD_SCHEMA_ONLY) { throw new UnsupportedOperationException("Local relationship tables cannot be used in OLD_SCHEMA_ONLY mode, so they cannot be backfilled."); } diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java index f8c1ff770..8d70df236 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java @@ -382,7 +382,7 @@ public Task backfillRelationshipTables(@ActionParam(PARAM_URNS) .setRelationship(relationship.getClass().getSimpleName()); backfillResult.getRelationships().add(backfillResultRelationship); - } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } }); From f6d2f9051cc73c8cb1230409376ef9ee2ce93b5e Mon Sep 17 00:00:00 2001 From: Jesse Jia Date: Mon, 16 Oct 2023 14:15:23 -0700 Subject: [PATCH 5/5] fix style --- .../java/com/linkedin/metadata/restli/BaseEntityResource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java index 8d70df236..835592256 100644 --- a/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java +++ b/restli-resources/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java @@ -36,7 +36,6 @@ import com.linkedin.restli.server.annotations.QueryParam; import com.linkedin.restli.server.annotations.RestMethod; import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; -import java.lang.reflect.InvocationTargetException; import java.time.Clock; import java.util.ArrayList; import java.util.Arrays;