diff --git a/dao-impl/ebean-dao/build.gradle b/dao-impl/ebean-dao/build.gradle index 8ddc38eb0..633965677 100644 --- a/dao-impl/ebean-dao/build.gradle +++ b/dao-impl/ebean-dao/build.gradle @@ -9,6 +9,7 @@ configurations { dependencies { compile project(':core-models-utils') compile project(':dao-api') + compile project(':gradle-plugins:metadata-annotations-lib') compile externalDependency.ebean compile externalDependency.flywayCore compile externalDependency.guava @@ -20,6 +21,7 @@ dependencies { annotationProcessor externalDependency.lombok + testCompile project(':gradle-plugins:metadata-annotations-test-models') testCompile project(':testing:core-models-testing') testCompile project(':testing:test-models') testCompile externalDependency.mysql diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java index 951f56461..83ce335e0 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/EBeanDAOUtils.java @@ -4,7 +4,11 @@ import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.annotations.AlwaysAllowList; import com.linkedin.metadata.annotations.DeltaEntityAnnotationArrayMap; +import com.linkedin.metadata.annotations.GmaAnnotation; +import com.linkedin.metadata.annotations.GmaAnnotationParser; +import com.linkedin.metadata.annotations.ModelType; import com.linkedin.metadata.aspect.AuditedAspect; import com.linkedin.metadata.aspect.SoftDeletedAspect; import com.linkedin.metadata.dao.EbeanMetadataAspect; @@ -27,10 +31,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; @@ -366,4 +373,50 @@ public static LocalRelationshipCriterion buildRelationshipFieldCriterion(LocalRe .setValue(localRelationshipValue) .setCondition(condition); } + + /** + * Extract the non-null values from all top-level relationship fields of an aspect. + * @param aspect aspect (possibly with relationships) to be ingested + * @return a list of relationship arrays, with each array representing the relationship(s) present in each top-level relationship + * field in the aspect. An empty list means that there is no non-null relationship metadata attached to the given aspect. + */ + @Nonnull + public static List> extractRelationshipsFromAspect(ASPECT aspect) { + return aspect.schema().getFields().stream().filter(field -> !field.getType().isPrimitive()).map(field -> { + String fieldName = field.getName(); + Class clazz = (Class) aspect.getClass(); + try { + Method getMethod = clazz.getMethod("get" + StringUtils.capitalize(fieldName)); // getFieldName + Object obj = getMethod.invoke(aspect); // invoke getFieldName + // all relationship fields will be represented as Arrays so filter out any non-lists, empty lists, and lists that don't contain RecordTemplates + if (!(obj instanceof List) || ((List) obj).isEmpty() || !(((List) obj).get(0) instanceof RecordTemplate)) { + return null; + } + List relationshipsList = (List) obj; + ModelType modelType = parseModelTypeFromGmaAnnotation(relationshipsList.get(0)); + if (modelType == ModelType.RELATIONSHIP) { + log.debug(String.format("Found {%d} relationships of type {%s} for field {%s} of aspect class {%s}.", + relationshipsList.size(), relationshipsList.get(0).getClass(), fieldName, clazz.getName())); + return (List) relationshipsList; + } + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + return null; + }).filter(Objects::nonNull).collect(Collectors.toList()); + } + + // Using the GmaAnnotationParser, extract the model type from the @gma.model annotation on any models. + private static ModelType parseModelTypeFromGmaAnnotation(RecordTemplate model) { + try { + final RecordDataSchema schema = (RecordDataSchema) DataTemplateUtil.getSchema(model.getClass()); + final Optional gmaAnnotation = new GmaAnnotationParser(new AlwaysAllowList()).parse(schema); + if (!gmaAnnotation.isPresent() || !gmaAnnotation.get().hasModel()) { + return null; + } + return gmaAnnotation.get().getModel(); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to parse the annotations for field %s", model.getClass().getCanonicalName()), e); + } + } } diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EBeanDAOUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EBeanDAOUtilsTest.java index a6b909296..a1272e581 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EBeanDAOUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/EBeanDAOUtilsTest.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.dao.utils; import com.google.common.io.Resources; +import com.linkedin.data.template.IntegerArray; +import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.metadata.aspect.AuditedAspect; import com.linkedin.metadata.dao.EbeanLocalAccess; @@ -13,7 +15,15 @@ import com.linkedin.metadata.query.LocalRelationshipValue; import com.linkedin.metadata.query.RelationshipField; import com.linkedin.metadata.query.UrnField; +import com.linkedin.testing.AnnotatedAspectBarWithRelationshipFields; +import com.linkedin.testing.AnnotatedRelationshipBar; +import com.linkedin.testing.AnnotatedRelationshipBarArray; +import com.linkedin.testing.AnnotatedRelationshipFoo; +import com.linkedin.testing.AnnotatedRelationshipFooArray; import com.linkedin.testing.AspectFoo; +import com.linkedin.testing.AnnotatedAspectFooWithRelationshipField; +import com.linkedin.testing.CommonAspect; +import com.linkedin.testing.CommonAspectArray; import com.linkedin.testing.urn.BurgerUrn; import com.linkedin.testing.urn.FooUrn; import io.ebean.Ebean; @@ -567,4 +577,62 @@ public void testBuildRelationshipFieldCriterionWithAspectField() { relationshipField); assertEquals(relationshipField, filterCriterion.getField().getRelationshipField()); } + + @Test + public void testExtractRelationshipsFromAspect() { + // case 1: aspect model does not contain any relationship typed fields + // expected: return null + AspectFoo foo = new AspectFoo(); + assertTrue(EBeanDAOUtils.extractRelationshipsFromAspect(foo).isEmpty()); + + // case 2: aspect model contains only a non-null relationship type field + // expected: return the non-null relationship + AnnotatedRelationshipFoo relationshipFoo = new AnnotatedRelationshipFoo(); + AnnotatedRelationshipFooArray relationshipFoos = new AnnotatedRelationshipFooArray(relationshipFoo); + AnnotatedAspectFooWithRelationshipField fooWithRelationshipField = new AnnotatedAspectFooWithRelationshipField() + .setRelationshipFoo(relationshipFoos); + + List> results = EBeanDAOUtils.extractRelationshipsFromAspect(fooWithRelationshipField); + assertEquals(1, results.size()); + assertEquals(1, results.get(0).size()); + assertEquals(relationshipFoo, results.get(0).get(0)); + + // case 3: aspect model contains only a null relationship type field + // expected: return null + AnnotatedAspectFooWithRelationshipField fooWithNullRelationshipField = new AnnotatedAspectFooWithRelationshipField(); + assertTrue(EBeanDAOUtils.extractRelationshipsFromAspect(fooWithNullRelationshipField).isEmpty()); + + // case 4: aspect model contains multiple relationship fields, some null and some non-null, as well as array fields + // containing non-Relationship objects + // expected: return only the non-null relationships + relationshipFoos = new AnnotatedRelationshipFooArray(new AnnotatedRelationshipFoo(), new AnnotatedRelationshipFoo()); + AnnotatedRelationshipBarArray relationshipBars = new AnnotatedRelationshipBarArray(new AnnotatedRelationshipBar()); + // given: + // aspect = { + // value -> "abc" + // integers -> [1] + // nonRelationshipStructs -> [commonAspect1] + // relationshipFoos -> [foo1, foo2] + // relationshipBars -> [bar1] + // moreRelationshipFoos -> not present + // } + // expect: + // [[foo1, foo2], [bar1]] + AnnotatedAspectBarWithRelationshipFields barWithRelationshipFields = new AnnotatedAspectBarWithRelationshipFields() + .setValue("abc") + .setIntegers(new IntegerArray(1)) + .setNonRelationshipStructs(new CommonAspectArray(new CommonAspect())) + .setRelationshipFoos(relationshipFoos) + .setRelationshipBars(relationshipBars); // don't set moreRelationshipFoos field + + results = EBeanDAOUtils.extractRelationshipsFromAspect(barWithRelationshipFields); + assertEquals(2, results.size()); + assertEquals(2, results.get(0).size()); + assertEquals(1, results.get(1).size()); + assertEquals(new AnnotatedRelationshipFoo(), results.get(0).get(0)); + assertEquals(new AnnotatedRelationshipFoo(), results.get(0).get(1)); + assertEquals(new AnnotatedRelationshipBar(), results.get(1).get(0)); + } + + } \ No newline at end of file diff --git a/gradle-plugins/metadata-annotations-schema/src/main/pegasus/com/linkedin/metadata/annotations/GmaAnnotation.pdl b/gradle-plugins/metadata-annotations-schema/src/main/pegasus/com/linkedin/metadata/annotations/GmaAnnotation.pdl index b4ba870be..bf4fa9f4f 100644 --- a/gradle-plugins/metadata-annotations-schema/src/main/pegasus/com/linkedin/metadata/annotations/GmaAnnotation.pdl +++ b/gradle-plugins/metadata-annotations-schema/src/main/pegasus/com/linkedin/metadata/annotations/GmaAnnotation.pdl @@ -14,6 +14,15 @@ record GmaAnnotation { */ delta: optional DeltaAnnotation + /** + * The type of model + */ + model: optional enum ModelType { + ASSET, + ASPECT, + RELATIONSHIP + } + /** * Information about GMA Search functionality. */ diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/java/com/linkedin/testing/urn/BarUrn.java b/gradle-plugins/metadata-annotations-test-models/src/main/java/com/linkedin/testing/urn/BarUrn.java index 2ddc6826b..06241b5d4 100644 --- a/gradle-plugins/metadata-annotations-test-models/src/main/java/com/linkedin/testing/urn/BarUrn.java +++ b/gradle-plugins/metadata-annotations-test-models/src/main/java/com/linkedin/testing/urn/BarUrn.java @@ -6,7 +6,7 @@ public final class BarUrn extends Urn { - public static final String ENTITY_TYPE = "entityBar"; + public static final String ENTITY_TYPE = "bar"; public BarUrn(int id) throws URISyntaxException { super(ENTITY_TYPE, Integer.toString(id)); diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectBarWithRelationshipFields.pdl b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectBarWithRelationshipFields.pdl new file mode 100644 index 000000000..55466e65d --- /dev/null +++ b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectBarWithRelationshipFields.pdl @@ -0,0 +1,38 @@ +namespace com.linkedin.testing + +/** + * For unit tests + */ +@gma.aspect.column.name = "annotatedaspectbarwithrelationshipfields" +@gma.model = "ASPECT" +record AnnotatedAspectBarWithRelationshipFields { + /** + * For unit tests + */ + value: string + + /** + * For unit tests + */ + integers: optional array[int] + + /** + * For unit tests + */ + nonRelationshipStructs: optional array[CommonAspect] + + /** + * For unit tests + */ + relationshipFoos: optional array[AnnotatedRelationshipFoo] + + /** + * For unit tests + */ + relationshipBars: optional array[AnnotatedRelationshipBar] + + /** + * For unit tests + */ + moreRelationshipFoos: optional array[AnnotatedRelationshipFoo] +} \ No newline at end of file diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectFooWithRelationshipField.pdl b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectFooWithRelationshipField.pdl new file mode 100644 index 000000000..1faaebabe --- /dev/null +++ b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedAspectFooWithRelationshipField.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.testing + +/** + * For unit tests + */ +@gma.aspect.column.name = "annotatedaspectfoowithrelationshipfield" +@gma.model = "ASPECT" +record AnnotatedAspectFooWithRelationshipField { + /** + * For unit tests + */ + value: string + + /** + * For unit tests + */ + relationshipFoo: optional array[AnnotatedRelationshipFoo] +} \ No newline at end of file diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipBar.pdl b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipBar.pdl new file mode 100644 index 000000000..956deed96 --- /dev/null +++ b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipBar.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.testing + +import com.linkedin.common.Urn + +/** + * For unit tests + */ +@pairings = [ { + "destination" : "com.linkedin.common.urn.Urn", + "source" : "com.linkedin.common.urn.Urn" +} ] +@gma.model = "RELATIONSHIP" +record AnnotatedRelationshipBar { + + /** + * For unit tests + */ + source: Urn + + /** + * For unit tests + */ + destination: Urn +} \ No newline at end of file diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipFoo.pdl b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipFoo.pdl new file mode 100644 index 000000000..d31a3f854 --- /dev/null +++ b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/AnnotatedRelationshipFoo.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.testing + +import com.linkedin.common.Urn + +/** + * For unit tests + */ +@pairings = [ { + "destination" : "com.linkedin.common.urn.Urn", + "source" : "com.linkedin.common.urn.Urn" +} ] +@gma.model = "RELATIONSHIP" +record AnnotatedRelationshipFoo { + + /** + * For unit tests + */ + source: Urn + + /** + * For unit tests + */ + destination: Urn +} \ No newline at end of file diff --git a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl index e19576630..135e58c48 100644 --- a/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl +++ b/gradle-plugins/metadata-annotations-test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl @@ -9,7 +9,7 @@ namespace com.linkedin.testing } @validate.`com.linkedin.common.validator.TypedUrnValidator` = { "owningTeam" : "urn:li:internalTeam:metadata", - "entityType" : "entityBar", + "entityType" : "bar", "namespace" : "li", "name" : "Bar", "doc" : "Bar identifier.", diff --git a/testing/test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl b/testing/test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl index e19576630..135e58c48 100644 --- a/testing/test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl +++ b/testing/test-models/src/main/pegasus/com/linkedin/testing/BarUrn.pdl @@ -9,7 +9,7 @@ namespace com.linkedin.testing } @validate.`com.linkedin.common.validator.TypedUrnValidator` = { "owningTeam" : "urn:li:internalTeam:metadata", - "entityType" : "entityBar", + "entityType" : "bar", "namespace" : "li", "name" : "Bar", "doc" : "Bar identifier.", diff --git a/testing/test-models/src/main/pegasus/com/linkedin/testing/FooUrn.pdl b/testing/test-models/src/main/pegasus/com/linkedin/testing/FooUrn.pdl index d7ac15415..970d726cd 100644 --- a/testing/test-models/src/main/pegasus/com/linkedin/testing/FooUrn.pdl +++ b/testing/test-models/src/main/pegasus/com/linkedin/testing/FooUrn.pdl @@ -9,7 +9,7 @@ namespace com.linkedin.testing } @validate.`com.linkedin.common.validator.TypedUrnValidator` = { "owningTeam" : "urn:li:internalTeam:metadata", - "entityType" : "entityFoo", + "entityType" : "foo", "namespace" : "li", "name" : "Foo", "doc" : "Foo identifier.",