- * The invoked function takes a RequestScope as parameter. - * - */ -@Target({ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -@Deprecated -public @interface OnReadPostCommit { - /** - * Field name on which the annotated method is only triggered if that field is read. - * If value is empty string, then trigger once when the object is read. - * If value is "*", then trigger for all field reads. - * - * @return the field name that triggers the method - */ - String value() default ""; -} diff --git a/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreCommit.java b/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreCommit.java deleted file mode 100644 index 8291db6046..0000000000 --- a/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreCommit.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2016, Yahoo Inc. - * Licensed under the Apache License, Version 2.0 - * See LICENSE file in project root for terms. - */ -package com.yahoo.elide.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Pre-read hook. This annotation marks a callback that is triggered when a user performs a "read" action. - * This hook will be triggered after all security checks have been run, but before the datastore - * has been committed. - *
- * The invoked function takes a RequestScope as parameter. - * - */ -@Target({ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -@Deprecated -public @interface OnReadPreCommit { - /** - * Field name on which the annotated method is only triggered if that field is read. - * If value is empty string, then trigger once when the object is read. - * If value is "*", then trigger for all field reads. - * - * @return the field name that triggers the method - */ - String value() default ""; -} diff --git a/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreSecurity.java b/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreSecurity.java deleted file mode 100644 index a80bf9475e..0000000000 --- a/elide-core/src/main/java/com/yahoo/elide/annotation/OnReadPreSecurity.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2016, Yahoo Inc. - * Licensed under the Apache License, Version 2.0 - * See LICENSE file in project root for terms. - */ -package com.yahoo.elide.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * On read trigger annotation. - *
- * The invoked function takes a RequestScope as parameter. - * - */ -@Target({ElementType.METHOD}) -@Retention(RetentionPolicy.RUNTIME) -@Deprecated -public @interface OnReadPreSecurity { - /** - * Field name on which the annotated method is only triggered if that field is read. - * If value is empty string, then trigger once when the object is read. - * If value is "*", then trigger for all field reads. - * - * @return the field name that triggers this method - */ - String value() default ""; -} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/Path.java b/elide-core/src/main/java/com/yahoo/elide/core/Path.java index 44c6741e18..970d78eabc 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/Path.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/Path.java @@ -1,5 +1,5 @@ /* - * Copyright 2017, Oath Inc. + * Copyright 2017, Yahoo Inc. * Licensed under the Apache License, Version 2.0 * See LICENSE file in project root for terms. */ @@ -92,6 +92,18 @@ public Path(Type entityClass, EntityDictionary dictionary, String fieldName, pathElements = Lists.newArrayList(resolvePathAttribute(entityClass, fieldName, alias, arguments, dictionary)); } + public boolean isComputed(EntityDictionary dictionary) { + for (Path.PathElement pathElement : getPathElements()) { + Type entityClass = pathElement.getType(); + String fieldName = pathElement.getFieldName(); + + if (dictionary.isComputed(entityClass, fieldName)) { + return true; + } + } + return false; + } + /** * Resolve a dot separated path into list of path elements. * diff --git a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java index 1c9e99bcac..af1077d34f 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java @@ -7,8 +7,9 @@ import static com.yahoo.elide.annotation.LifeCycleHookBinding.Operation.CREATE; import static com.yahoo.elide.annotation.LifeCycleHookBinding.Operation.DELETE; -import static com.yahoo.elide.annotation.LifeCycleHookBinding.Operation.READ; import static com.yahoo.elide.annotation.LifeCycleHookBinding.Operation.UPDATE; +import static com.yahoo.elide.core.dictionary.EntityBinding.EMPTY_BINDING; +import static com.yahoo.elide.core.dictionary.EntityDictionary.getType; import static com.yahoo.elide.core.type.ClassType.COLLECTION_TYPE; import com.yahoo.elide.annotation.Audit; @@ -21,7 +22,9 @@ import com.yahoo.elide.core.audit.InvalidSyntaxException; import com.yahoo.elide.core.audit.LogMessage; import com.yahoo.elide.core.audit.LogMessageImpl; +import com.yahoo.elide.core.datastore.DataStoreIterable; import com.yahoo.elide.core.datastore.DataStoreTransaction; +import com.yahoo.elide.core.dictionary.EntityBinding; import com.yahoo.elide.core.dictionary.EntityDictionary; import com.yahoo.elide.core.dictionary.RelationshipType; import com.yahoo.elide.core.exceptions.BadRequestException; @@ -45,7 +48,9 @@ import com.yahoo.elide.core.type.ClassType; import com.yahoo.elide.core.type.Type; import com.yahoo.elide.core.utils.coerce.CoerceUtil; +import com.yahoo.elide.jsonapi.document.processors.WithMetadata; import com.yahoo.elide.jsonapi.models.Data; +import com.yahoo.elide.jsonapi.models.Meta; import com.yahoo.elide.jsonapi.models.Relationship; import com.yahoo.elide.jsonapi.models.Resource; import com.yahoo.elide.jsonapi.models.ResourceIdentifier; @@ -68,6 +73,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -124,7 +130,7 @@ public PersistentResource( @NonNull RequestScope scope ) { this.obj = obj; - this.type = EntityDictionary.getType(obj); + this.type = getType(obj); this.uuid = Optional.ofNullable(id); this.lineage = parent != null ? new ResourceLineage(parent.lineage, parent, parentRelationship) @@ -496,7 +502,7 @@ private static ExpressionResult checkPermission( private static ExpressionResult checkUserPermission( Class annotationClass, Object obj, RequestScope requestScope, Set requestedFields) { return requestScope.getPermissionExecutor() - .checkUserPermissions(EntityDictionary.getType(obj), annotationClass, requestedFields); + .checkUserPermissions(getType(obj), annotationClass, requestedFields); } protected static boolean checkIncludeSparseField(Map> sparseFields, String type, @@ -580,18 +586,31 @@ public boolean updateAttribute(String fieldName, Object newVal) { if (!Objects.equals(val, coercedNewValue)) { if (val == null || coercedNewValue == null - || !dictionary.isComplexAttribute(EntityDictionary.getType(obj), fieldName)) { + || !dictionary.isComplexAttribute(getType(obj), fieldName)) { this.setValueChecked(fieldName, coercedNewValue); } else { if (newVal instanceof Map) { - this.updateComplexAttribute(dictionary, (Map) newVal, val, requestScope); + + //We perform a copy here for two reasons: + //1. We want the original so we can dispatch update life cycle hooks. + //2. Some stores (Hibernate) won't notice changes to an attribute if the attribute + //has a @TypeDef annotation unless we modify the reference in the parent object. This rules + //out an update in place strategy. + Object copy = copyComplexAttribute(val); + + //Update the copy. + this.updateComplexAttribute(dictionary, (Map) newVal, copy, requestScope); + + //Set the copy. + dictionary.setValue(obj, fieldName, copy); + triggerUpdate(fieldName, val, copy); } else { this.setValueChecked(fieldName, coercedNewValue); } } this.markDirty(); //Hooks for customize logic for setAttribute/Relation - if (dictionary.isAttribute(EntityDictionary.getType(obj), fieldName)) { + if (dictionary.isAttribute(getType(obj), fieldName)) { transaction.setAttribute(obj, Attribute.builder() .name(fieldName) .type(fieldClass) @@ -630,6 +649,43 @@ private void updateComplexAttribute(EntityDictionary dictionary, } } + /** + * Copies a complex attribute. If the attribute fields are complex, recurses to perform a deep copy. + * @param object The attribute to copy. + * @return The copy. + */ + private Object copyComplexAttribute(Object object) { + if (object == null) { + return null; + } + + Type type = getType(object); + EntityBinding binding = dictionary.getEntityBinding(type); + + Preconditions.checkState(! binding.equals(EMPTY_BINDING), "Model not found."); + Preconditions.checkState(binding.apiRelationships.isEmpty(), "Deep copy of relationships not supported"); + + Object copy; + try { + copy = type.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalStateException("Cannot perform deep copy of " + type.getName(), e); + } + + binding.apiAttributes.forEach(attribute -> { + Object newValue; + Object oldValue = dictionary.getValue(object, attribute, requestScope); + if (! dictionary.isComplexAttribute(type, attribute)) { + newValue = oldValue; + } else { + newValue = copyComplexAttribute(oldValue); + } + dictionary.setValue(copy, attribute, newValue); + }); + + return copy; + } + /** * Perform a full replacement on relationships. * Here is an example: @@ -724,31 +780,24 @@ protected boolean updateToManyRelation(String fieldName, checkTransferablePermission(added); - Collection collection = (Collection) this.getValueUnchecked(fieldName); - - if (collection == null) { - this.setValue(fieldName, mine); - } - Set newRelationships = new LinkedHashSet<>(); Set deletedRelationships = new LinkedHashSet<>(); deleted .stream() .forEach(toDelete -> { - delFromCollection(collection, fieldName, toDelete, false); - deleteInverseRelation(fieldName, toDelete.getObject()); deletedRelationships.add(toDelete.getObject()); }); added .stream() .forEach(toAdd -> { - addToCollection(collection, fieldName, toAdd); - addInverseRelation(fieldName, toAdd.getObject()); newRelationships.add(toAdd.getObject()); }); + Collection collection = (Collection) this.getValueUnchecked(fieldName); + modifyCollection(collection, fieldName, newRelationships, deletedRelationships, true); + if (!updated.isEmpty()) { this.markDirty(); } @@ -830,14 +879,6 @@ public boolean clearRelation(String relationName) { RelationshipType type = getRelationshipType(relationName); - mine.stream() - .forEach(toDelete -> { - if (hasInverseRelation(relationName)) { - deleteInverseRelation(relationName, toDelete.getObject()); - toDelete.markDirty(); - } - }); - if (type.isToOne()) { PersistentResource oldValue = IterableUtils.first(mine); if (oldValue != null && oldValue.getObject() != null) { @@ -854,12 +895,9 @@ public boolean clearRelation(String relationName) { Set deletedRelationships = new LinkedHashSet<>(); mine.stream() .forEach(toDelete -> { - delFromCollection(collection, relationName, toDelete, false); - if (hasInverseRelation(relationName)) { - toDelete.markDirty(); - } deletedRelationships.add(toDelete.getObject()); }); + modifyCollection(collection, relationName, Collections.emptySet(), deletedRelationships, true); this.markDirty(); //hook for updateToManyRelation transaction.updateToManyRelation(transaction, obj, relationName, @@ -900,18 +938,19 @@ public void removeRelation(String fieldName, PersistentResource removeResource) //Nothing to do return; } - delFromCollection((Collection) relation, fieldName, removeResource, false); + modifyCollection((Collection) relation, fieldName, Collections.emptySet(), + Set.of(removeResource.getObject()), true); } else { if (relation == null || removeResource == null || !relation.equals(removeResource.getObject())) { //Nothing to do return; } this.nullValue(fieldName, removeResource); - } - if (hasInverseRelation(fieldName)) { - deleteInverseRelation(fieldName, removeResource.getObject()); - removeResource.markDirty(); + if (hasInverseRelation(fieldName)) { + deleteInverseRelation(fieldName, removeResource.getObject()); + removeResource.markDirty(); + } } if (!Objects.equals(original, modified)) { @@ -929,28 +968,6 @@ public void removeRelation(String fieldName, PersistentResource removeResource) } } - /** - * Checks whether the new entity is already a part of the relationship. - * - * @param fieldName which relation link - * @param toAdd the new relation - * @return boolean representing the result of the check - * - * This method does not handle the case where the toAdd entity is newly created, - * since newly created entities have null id there is no easy way to check for identity - */ - public boolean relationshipAlreadyExists(String fieldName, PersistentResource toAdd) { - Object relation = this.getValueUnchecked(fieldName); - String toAddId = toAdd.getId(); - if (toAddId == null) { - return false; - } - if (relation instanceof Collection) { - return ((Collection) relation).stream().anyMatch(obj -> toAddId.equals(dictionary.getId(obj))); - } - return toAddId.equals(dictionary.getId(relation)); - } - /** * Add relation link from a given parent resource to a child resource. * @@ -958,21 +975,20 @@ public boolean relationshipAlreadyExists(String fieldName, PersistentResource to * @param newRelation the new relation */ public void addRelation(String fieldName, PersistentResource newRelation) { - if (!newRelation.isNewlyCreated() && relationshipAlreadyExists(fieldName, newRelation)) { - return; - } checkTransferablePermission(Collections.singleton(newRelation)); Object relation = this.getValueUnchecked(fieldName); if (relation instanceof Collection) { - if (addToCollection((Collection) relation, fieldName, newRelation)) { + if (modifyCollection((Collection) relation, fieldName, + Set.of(newRelation.getObject()), Collections.emptySet(), true)) { this.markDirty(); - } - //Hook for updateToManyRelation - transaction.updateToManyRelation(transaction, obj, fieldName, - Sets.newHashSet(newRelation.getObject()), new LinkedHashSet<>(), requestScope); - addInverseRelation(fieldName, newRelation.getObject()); + //Hook for updateToManyRelation + transaction.updateToManyRelation(transaction, obj, fieldName, + Sets.newHashSet(newRelation.getObject()), new LinkedHashSet<>(), requestScope); + + addInverseRelation(fieldName, newRelation.getObject()); + } } else { // Not a collection, but may be trying to create a ToOne relationship. // NOTE: updateRelation marks dirty. @@ -1221,12 +1237,6 @@ private void assertPropertyExists(String propertyName) { private Observable getRelation(com.yahoo.elide.core.request.Relationship relationship, boolean checked) { - if (checked) { - //All getRelation calls funnel to here. We only publish events for actions triggered directly - //by the API client. - requestScope.publishLifecycleEvent(this, READ); - requestScope.publishLifecycleEvent(this, relationship.getName(), READ, Optional.empty()); - } if (checked && !checkRelation(relationship)) { return Observable.empty(); @@ -1324,19 +1334,21 @@ private Observable getRelationUnchecked( .build() ).build(); - Object val = transaction.getRelation(transaction, obj, modifiedRelationship, requestScope); - - if (val == null) { - return Observable.empty(); - } - Observable resources; - if (val instanceof Iterable) { - Iterable filteredVal = (Iterable) val; + if (type.isToMany()) { + DataStoreIterable val = transaction.getToManyRelation(transaction, obj, modifiedRelationship, requestScope); + + if (val == null) { + return Observable.empty(); + } resources = Observable.fromIterable( - new PersistentResourceSet(this, relationName, filteredVal, requestScope)); + new PersistentResourceSet(this, relationName, val, requestScope)); } else { + Object val = transaction.getToOneRelation(transaction, obj, modifiedRelationship, requestScope); + if (val == null) { + return Observable.empty(); + } resources = Observable.fromArray(new PersistentResource(val, this, relationName, requestScope.getUUIDFor(val), requestScope)); } @@ -1409,7 +1421,7 @@ public void setObject(T obj) { @Override @JsonIgnore public Type getResourceType() { - return (Type) dictionary.lookupBoundClass(EntityDictionary.getType(obj)); + return (Type) dictionary.lookupBoundClass(getType(obj)); } /** @@ -1544,6 +1556,26 @@ public Resource toResource(final Map relationships, if (requestScope.getElideSettings().isEnableJsonLinks()) { resource.setLinks(requestScope.getElideSettings().getJsonApiLinks().getResourceLevelLinks(this)); } + + if (! (getObject() instanceof WithMetadata)) { + return resource; + } + + WithMetadata withMetadata = (WithMetadata) getObject(); + Set fields = withMetadata.getMetadataFields(); + + if (fields.size() == 0) { + return resource; + } + + Meta meta = new Meta(new HashMap<>()); + + for (String field : fields) { + meta.getMetaMap().put(field, withMetadata.getMetadataField(field).get()); + } + + resource.setMeta(meta); + return resource; } @@ -1677,8 +1709,6 @@ protected void nullValue(String fieldName, PersistentResource oldValue) { * @return value value */ protected Object getValueChecked(Attribute attribute) { - requestScope.publishLifecycleEvent(this, READ); - requestScope.publishLifecycleEvent(this, attribute.getName(), READ, Optional.empty()); checkFieldAwareDeferPermissions(ReadPermission.class, attribute.getName(), null, null); return transaction.getAttribute(getObject(), attribute, requestScope); } @@ -1693,94 +1723,47 @@ protected Object getValueUnchecked(String fieldName) { return getValue(getObject(), fieldName, requestScope); } - /** - * Adds a new element to a collection and tests update permission. - * - * @param collection the collection - * @param collectionName the collection name - * @param toAdd the to add - * @return True if added to collection false otherwise (i.e. element already in collection) - */ - protected boolean addToCollection(Collection collection, String collectionName, PersistentResource toAdd) { - final Collection singleton = Collections.singleton(toAdd.getObject()); - final Collection original = copyCollection(collection); - checkFieldAwareDeferPermissions( - UpdatePermission.class, - collectionName, - CollectionUtils.union(CollectionUtils.emptyIfNull(collection), singleton), - original); - if (collection == null) { - collection = Collections.singleton(toAdd.getObject()); - Object value = getValueUnchecked(collectionName); - if (!Objects.equals(value, toAdd.getObject())) { - this.setValueChecked(collectionName, collection); - return true; - } - } else { - if (!collection.contains(toAdd.getObject())) { - collection.add(toAdd.getObject()); + protected boolean modifyCollection( + Collection toModify, + String collectionName, + Collection toAdd, + Collection toRemove, + boolean updateInverse) { - triggerUpdate(collectionName, original, collection); - return true; - } - } - return false; - } + Collection copyOfOriginal = copyCollection(toModify); + + Collection modified = CollectionUtils.union(CollectionUtils.emptyIfNull(toModify), toAdd); + modified = CollectionUtils.subtract(modified, toRemove); - /** - * Deletes an existing element in a collection and tests update and delete permissions. - * - * @param collection the collection - * @param collectionName the collection name - * @param toDelete the to delete - * @param isInverseCheck Whether or not the deletion is already coming from cleaning up an inverse. - * Without this parameter, we could find ourselves in a loop of checks. - * TODO: This is a band-aid for a quick fix. This should certainly be refactored. - */ - protected void delFromCollection( - Collection collection, - String collectionName, - PersistentResource toDelete, - boolean isInverseCheck) { - final Collection original = copyCollection(collection); checkFieldAwareDeferPermissions( UpdatePermission.class, collectionName, - CollectionUtils.disjunction(collection, Collections.singleton(toDelete.getObject())), - original - ); - - String inverseField = getInverseRelationField(collectionName); - if (!isInverseCheck && !inverseField.isEmpty()) { - // Compute the ChangeSpec for the inverse relation and check whether or not we have access - // to apply this change to that field. - final Object originalValue = toDelete.getValueUnchecked(inverseField); - final Collection originalBidirectional; + modified, + copyOfOriginal); - if (originalValue instanceof Collection) { - originalBidirectional = copyCollection((Collection) originalValue); - } else { - originalBidirectional = Collections.singleton(originalValue); + if (updateInverse) { + for (Object adding : toAdd) { + addInverseRelation(collectionName, adding); } - final Collection removedBidrectional = CollectionUtils - .disjunction(Collections.singleton(this.getObject()), originalBidirectional); - - toDelete.checkFieldAwareDeferPermissions( - UpdatePermission.class, - inverseField, - removedBidrectional, - originalBidirectional - ); - } - - if (collection == null) { - return; + for (Object removing : toRemove) { + deleteInverseRelation(collectionName, removing); + } } - collection.remove(toDelete.getObject()); + if (toModify == null) { + this.setValueChecked(collectionName, modified); + return true; + } else { + if (copyOfOriginal.equals(modified)) { + return false; + } + toModify.addAll(toAdd); + toModify.removeAll(toRemove); - triggerUpdate(collectionName, original, collection); + triggerUpdate(collectionName, copyOfOriginal, modified); + return true; + } } /** @@ -1820,7 +1803,8 @@ protected void deleteInverseRelation(String relationName, Object inverseEntity) } if (inverseRelation instanceof Collection) { - inverseResource.delFromCollection((Collection) inverseRelation, inverseField, this, true); + inverseResource.modifyCollection((Collection) inverseRelation, inverseField, + Collections.emptySet(), Set.of(this.getObject()), false); } else if (inverseType.isAssignableFrom(this.getResourceType())) { inverseResource.nullValue(inverseField, this); } else { @@ -1870,7 +1854,8 @@ protected void addInverseRelation(String relationName, Object inverseObj) { if (COLLECTION_TYPE.isAssignableFrom(inverseType)) { if (inverseRelation != null) { - inverseResource.addToCollection((Collection) inverseRelation, inverseName, this); + inverseResource.modifyCollection((Collection) inverseRelation, inverseName, + Set.of(this.getObject()), Collections.emptySet(), false); } else { inverseResource.setValueChecked(inverseName, Collections.singleton(this.getObject())); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java index c1ccdb8213..c32ae0854c 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.core; +import static com.yahoo.elide.annotation.LifeCycleHookBinding.ALL_OPERATIONS; import com.yahoo.elide.ElideSettings; import com.yahoo.elide.annotation.LifeCycleHookBinding; import com.yahoo.elide.core.audit.AuditLogger; @@ -27,9 +28,6 @@ import com.yahoo.elide.jsonapi.JsonApiMapper; import com.yahoo.elide.jsonapi.models.JsonApiDocument; import org.apache.commons.collections.MapUtils; -import io.reactivex.Observable; -import io.reactivex.subjects.PublishSubject; -import io.reactivex.subjects.ReplaySubject; import lombok.Getter; import lombok.Setter; @@ -76,9 +74,9 @@ public class RequestScope implements com.yahoo.elide.core.security.RequestScope @Getter private final UUID requestId; private final Map expressionsByType; - private PublishSubject lifecycleEvents; - private Observable distinctLifecycleEvents; - private ReplaySubject queuedLifecycleEvents; + private final Map metadata; + + private LinkedHashSet eventQueue; /* Used to filter across heterogeneous types during the first load */ private FilterExpression globalFilterExpression; @@ -107,10 +105,7 @@ public RequestScope(String baseUrlEndPoint, UUID requestId, ElideSettings elideSettings) { this.apiVersion = apiVersion; - this.lifecycleEvents = PublishSubject.create(); - this.distinctLifecycleEvents = lifecycleEvents.distinct(); - this.queuedLifecycleEvents = ReplaySubject.create(); - this.distinctLifecycleEvents.subscribe(queuedLifecycleEvents); + this.eventQueue = new LinkedHashSet<>(); this.path = path; this.baseUrlEndPoint = baseUrlEndPoint; @@ -132,12 +127,12 @@ public RequestScope(String baseUrlEndPoint, this.dirtyResources = new LinkedHashSet<>(); this.deletedResources = new LinkedHashSet<>(); this.requestId = requestId; + this.metadata = new HashMap<>(); this.queryParams = queryParams == null ? new MultivaluedHashMap<>() : queryParams; this.requestHeaders = MapUtils.isEmpty(requestHeaders) ? Collections.emptyMap() : requestHeaders; - registerPreSecurityObservers(); this.sparseFields = parseSparseFields(getQueryParams()); @@ -218,12 +213,11 @@ protected RequestScope(String path, String apiVersion, this.filterDialect = outerRequestScope.filterDialect; this.expressionsByType = outerRequestScope.expressionsByType; this.elideSettings = outerRequestScope.elideSettings; - this.lifecycleEvents = outerRequestScope.lifecycleEvents; - this.distinctLifecycleEvents = outerRequestScope.distinctLifecycleEvents; + this.eventQueue = outerRequestScope.eventQueue; this.updateStatusCode = outerRequestScope.updateStatusCode; - this.queuedLifecycleEvents = outerRequestScope.queuedLifecycleEvents; this.requestId = outerRequestScope.requestId; this.sparseFields = outerRequestScope.sparseFields; + this.metadata = new HashMap<>(outerRequestScope.metadata); } public Set getNewResources() { @@ -326,115 +320,59 @@ private static MultivaluedMap getFilterParams(MultivaluedMap event.getEventType().equals(operation)) + .forEach(event -> { + invoker.onNext(event); + }); + } + /** * Run queued pre-security lifecycle triggers. */ public void runQueuedPreSecurityTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, false)) - .throwOnError(); + notifySubscribers(LifeCycleHookBinding.Operation.CREATE, LifeCycleHookBinding.TransactionPhase.PRESECURITY); } /** * Run queued pre-flush lifecycle triggers. */ public void runQueuedPreFlushTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); + runQueuedPreFlushTriggers(ALL_OPERATIONS); + } + + /** + * Run queued pre-flush lifecycle triggers. + * @param operations List of operations to run pre-flush triggers for. + */ + public void runQueuedPreFlushTriggers(LifeCycleHookBinding.Operation[] operations) { + for (LifeCycleHookBinding.Operation op : operations) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.PREFLUSH); + } } /** * Run queued pre-commit lifecycle triggers. */ public void runQueuedPreCommitTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); + for (LifeCycleHookBinding.Operation op : ALL_OPERATIONS) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.PRECOMMIT); + } } /** * Run queued post-commit lifecycle triggers. */ public void runQueuedPostCommitTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); + for (LifeCycleHookBinding.Operation op : ALL_OPERATIONS) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.POSTCOMMIT); + } } /** @@ -444,9 +382,7 @@ public void runQueuedPostCommitTriggers() { * @param crudAction CRUD action */ protected void publishLifecycleEvent(PersistentResource resource, LifeCycleHookBinding.Operation crudAction) { - lifecycleEvents.onNext( - new CRUDEvent(crudAction, resource, PersistentResource.CLASS_NO_FIELD, Optional.empty()) - ); + publishLifecycleEvent(new CRUDEvent(crudAction, resource, PersistentResource.CLASS_NO_FIELD, Optional.empty())); } /** @@ -461,9 +397,23 @@ protected void publishLifecycleEvent(PersistentResource resource, String fieldName, LifeCycleHookBinding.Operation crudAction, Optional changeSpec) { - lifecycleEvents.onNext( - new CRUDEvent(crudAction, resource, fieldName, changeSpec) - ); + publishLifecycleEvent(new CRUDEvent(crudAction, resource, fieldName, changeSpec)); + } + + protected void publishLifecycleEvent(CRUDEvent event) { + if (! eventQueue.contains(event)) { + if (event.getEventType().equals(LifeCycleHookBinding.Operation.DELETE) + || event.getEventType().equals(LifeCycleHookBinding.Operation.UPDATE)) { + + LifecycleHookInvoker invoker = new LifecycleHookInvoker(dictionary, + event.getEventType(), + LifeCycleHookBinding.TransactionPhase.PRESECURITY); + + invoker.onNext(event); + } + + eventQueue.add(event); + } } public void saveOrCreateObjects() { @@ -511,27 +461,6 @@ private String getInheritanceKey(String subClass, String superClass) { return subClass + "!" + superClass; } - private void registerPreSecurityObservers() { - - this.distinctLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - - this.distinctLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - - this.distinctLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - } - @Override public String getRequestHeaderByName(String headerName) { if (this.requestHeaders.get(headerName) == null) { @@ -539,4 +468,19 @@ public String getRequestHeaderByName(String headerName) { } return this.requestHeaders.get(headerName).get(0); } + + @Override + public void setMetadataField(String property, Object value) { + metadata.put(property, value); + } + + @Override + public Optional getMetadataField(String property) { + return Optional.ofNullable(metadata.getOrDefault(property, null)); + } + + @Override + public Set getMetadataFields() { + return metadata.keySet(); + } } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java b/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java index 291bb477a4..ab279214e7 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java @@ -8,15 +8,16 @@ import com.yahoo.elide.core.datastore.DataStoreTransaction; import lombok.Getter; -import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + /** * Transaction Registry class. */ @Getter public class TransactionRegistry { - private Map transactionMap = new HashMap<>(); + private Map transactionMap = new ConcurrentHashMap<>(); public Map getRunningTransactions() { return transactionMap; diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java new file mode 100644 index 0000000000..88c647dfd3 --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.core.datastore; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Consumer; + +/** + * Returns data loaded from a DataStore. Wraps an iterable but also communicates to Elide + * if the framework needs to filter, sort, or paginate the iterable in memory before returning to the client. + * @param The type being iterated over. + */ +public interface DataStoreIterable extends Iterable { + + /** + * Returns the underlying iterable. + * @return The underlying iterable. + */ + Iterable getWrappedIterable(); + + @Override + default Iterator iterator() { + return getWrappedIterable().iterator(); + } + + @Override + default void forEach(Consumer action) { + getWrappedIterable().forEach(action); + } + + @Override + default Spliterator spliterator() { + return Spliterators.spliteratorUnknownSize(iterator(), 0); + } + + /** + * Whether the iterable should be filtered in memory. + * @return true if the iterable needs sorting in memory. false otherwise. + */ + default boolean needsInMemoryFilter() { + return false; + } + + /** + * Whether the iterable should be sorted in memory. + * @return true if the iterable needs sorting in memory. false otherwise. + */ + default boolean needsInMemorySort() { + return false; + } + + /** + * Whether the iterable should be paginated in memory. + * @return true if the iterable needs pagination in memory. false otherwise. + */ + default boolean needsInMemoryPagination() { + return false; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java new file mode 100644 index 0000000000..ba12743df0 --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java @@ -0,0 +1,109 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.core.datastore; + +import java.util.ArrayList; + +/** + * Constructs DataStoreIterables. + * @param + */ +public class DataStoreIterableBuilder { + + private boolean filterInMemory = false; + private boolean sortInMemory = false; + private boolean paginateInMemory = false; + private final Iterable wrapped; + + /** + * Constructor. + */ + public DataStoreIterableBuilder() { + this.wrapped = new ArrayList<>(); + } + + /** + * Constructor. + * @param wrapped Required iterable to wrap. + */ + public DataStoreIterableBuilder(Iterable wrapped) { + if (wrapped == null) { + this.wrapped = new ArrayList<>(); + } else { + this.wrapped = wrapped; + } + } + + /** + * Filter the iterable in memory. + * @param filterInMemory true to filter in memory. + * @return the builder. + */ + public DataStoreIterableBuilder filterInMemory(boolean filterInMemory) { + this.filterInMemory = filterInMemory; + return this; + } + + /** + * Sorts the iterable in memory. + * @param sortInMemory true to sort in memory. + * @return the builder. + */ + public DataStoreIterableBuilder sortInMemory(boolean sortInMemory) { + this.sortInMemory = sortInMemory; + return this; + } + + /** + * Paginates the iterable in memory. + * @param paginateInMemory true to paginate in memory. + * @return the builder. + */ + public DataStoreIterableBuilder paginateInMemory(boolean paginateInMemory) { + this.paginateInMemory = paginateInMemory; + return this; + } + + /** + * Filter, sort, and paginate in memory. + * @return the builder. + */ + public DataStoreIterableBuilder allInMemory() { + this.filterInMemory = true; + this.sortInMemory = true; + this.paginateInMemory = true; + return this; + } + + /** + * Constructs the DataStoreIterable. + * @return the new iterable. + */ + public DataStoreIterable build() { + return new DataStoreIterable() { + @Override + public Iterable getWrappedIterable() { + return wrapped; + } + + @Override + public boolean needsInMemoryFilter() { + return filterInMemory; + } + + @Override + public boolean needsInMemorySort() { + return sortInMemory; + } + + @Override + public boolean needsInMemoryPagination() { + return paginateInMemory; + } + }; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java index b88dd0c3dc..22c5da62d3 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java @@ -23,21 +23,11 @@ import java.io.Closeable; import java.io.Serializable; import java.util.Iterator; -import java.util.Optional; import java.util.Set; /** * Wraps the Database Transaction type. */ public interface DataStoreTransaction extends Closeable { - - /** - * The extent to which the transaction supports a particular feature. - */ - public enum FeatureSupport { - FULL, - PARTIAL, - NONE - } /** * Save the updated object. * @@ -170,12 +160,12 @@ default T loadObject(EntityProjection entityProjection, * @param - The model type being loaded. * @return a collection of the loaded objects */ - Iterable loadObjects( + DataStoreIterable loadObjects( EntityProjection entityProjection, RequestScope scope); /** - * Retrieve a relation from an object. + * Retrieve a to-many relation from an object. * * @param relationTx - The datastore that governs objects of the relationhip's type. * @param entity - The object which owns the relationship. @@ -185,7 +175,28 @@ Iterable loadObjects( * @param - The model type of the relationship. * @return the object in the relation */ - default R getRelation( + default DataStoreIterable getToManyRelation( + DataStoreTransaction relationTx, + T entity, + Relationship relationship, + RequestScope scope) { + + return new DataStoreIterableBuilder( + (Iterable) PersistentResource.getValue(entity, relationship.getName(), scope)).allInMemory().build(); + } + + /** + * Retrieve a to-one relation from an object. + * + * @param relationTx - The datastore that governs objects of the relationhip's type. + * @param entity - The object which owns the relationship. + * @param relationship - the relationship to fetch. + * @param scope - contains request level metadata. + * @param - The model type which owns the relationship. + * @param - The model type of the relationship. + * @return the object in the relation + */ + default R getToOneRelation( DataStoreTransaction relationTx, T entity, Relationship relationship, @@ -270,48 +281,6 @@ default void setAttribute(T entity, RequestScope scope) { } - /** - * Whether or not the transaction can filter the provided class with the provided expression. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being filtered). - * @return FULL, PARTIAL, or NONE - */ - default FeatureSupport supportsFiltering(RequestScope scope, - Optional parent, - EntityProjection projection) { - return FeatureSupport.FULL; - } - - /** - * Whether or not the transaction can sort the provided class. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being sorted). - * @return true if sorting is possible - */ - default boolean supportsSorting(RequestScope scope, - Optional parent, - EntityProjection projection) { - return true; - } - - /** - * Whether or not the transaction can paginate the provided class. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being paginated). - * @return true if pagination is possible - */ - default boolean supportsPagination(RequestScope scope, - Optional parent, - EntityProjection projection) { - return true; - } - /** * Cancel running transaction. * Implementation must be thread-safe. diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java new file mode 100644 index 0000000000..d8d479e95b --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java @@ -0,0 +1,77 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.core.datastore.inmemory; + +import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.filter.expression.FilterExpression; +import com.yahoo.elide.core.filter.expression.InMemoryFilterExecutor; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.function.Predicate; + +/** + * An iterator which filters another iterator by an Elide filter expression. + * @param The type being iterated over. + */ +public class FilteredIterator implements Iterator { + + private Iterator wrapped; + private Predicate predicate; + + private T next; + + /** + * Constructor. + * @param filterExpression The filter expression to filter on. + * @param scope Request scope. + * @param wrapped The wrapped iterator. + */ + public FilteredIterator(FilterExpression filterExpression, RequestScope scope, Iterator wrapped) { + this.wrapped = wrapped; + InMemoryFilterExecutor executor = new InMemoryFilterExecutor(scope); + + predicate = filterExpression.accept(executor); + } + + @Override + public boolean hasNext() { + try { + next = next(); + } catch (NoSuchElementException e) { + return false; + } + + return true; + } + + @Override + public T next() { + if (next != null) { + T result = next; + next = null; + return result; + } + + while (next == null && wrapped.hasNext()) { + try { + next = wrapped.next(); + } catch (NoSuchElementException e) { + next = null; + } + if (next == null || ! predicate.test(next)) { + next = null; + } + } + + if (next == null) { + throw new NoSuchElementException(); + } + + return next; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java index a3520b5d00..45eee754e4 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java @@ -6,6 +6,8 @@ package com.yahoo.elide.core.datastore.inmemory; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; +import com.yahoo.elide.core.datastore.DataStoreIterableBuilder; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.dictionary.EntityDictionary; import com.yahoo.elide.core.exceptions.TransactionException; @@ -19,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import javax.persistence.GeneratedValue; @@ -125,19 +126,20 @@ public void setId(Object value, String id) { } @Override - public Object getRelation(DataStoreTransaction relationTx, - Object entity, - Relationship relationship, - RequestScope scope) { - return dictionary.getValue(entity, relationship.getName(), scope); + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, + Object entity, + Relationship relationship, + RequestScope scope) { + return new DataStoreIterableBuilder( + (Iterable) dictionary.getValue(entity, relationship.getName(), scope)).allInMemory().build(); } @Override - public Iterable loadObjects(EntityProjection projection, - RequestScope scope) { + public DataStoreIterable loadObjects(EntityProjection projection, + RequestScope scope) { synchronized (dataStore) { Map data = dataStore.get(projection.getType()); - return data.values(); + return new DataStoreIterableBuilder<>(data.values()).allInMemory().build(); } } @@ -163,21 +165,6 @@ public void close() throws IOException { operations.clear(); } - @Override - public FeatureSupport supportsFiltering(RequestScope scope, Optional parent, EntityProjection projection) { - return FeatureSupport.NONE; - } - - @Override - public boolean supportsSorting(RequestScope scope, Optional parent, EntityProjection projection) { - return false; - } - - @Override - public boolean supportsPagination(RequestScope scope, Optional parent, EntityProjection projection) { - return false; - } - private boolean containsObject(Object obj) { return containsObject(EntityDictionary.getType(obj), obj); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java index d3358737f7..4cf1da1aff 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java @@ -9,11 +9,12 @@ import com.yahoo.elide.core.Path; import com.yahoo.elide.core.PersistentResource; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; +import com.yahoo.elide.core.datastore.DataStoreIterableBuilder; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.FilterPredicatePushdownExtractor; import com.yahoo.elide.core.filter.expression.InMemoryExecutionVerifier; -import com.yahoo.elide.core.filter.expression.InMemoryFilterExecutor; import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; import com.yahoo.elide.core.request.Pagination; @@ -27,11 +28,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -64,24 +65,23 @@ public class InMemoryStoreTransaction implements DataStoreTransaction { */ @FunctionalInterface private interface DataFetcher { - Object fetch(Optional filterExpression, - Optional sorting, - Optional pagination, - RequestScope scope); + DataStoreIterable fetch(Optional filterExpression, + Optional sorting, + Optional pagination, + RequestScope scope); } - public InMemoryStoreTransaction(DataStoreTransaction tx) { this.tx = tx; } @Override - public Object getRelation(DataStoreTransaction relationTx, - Object entity, - Relationship relationship, - RequestScope scope) { + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, + Object entity, + Relationship relationship, + RequestScope scope) { DataFetcher fetcher = (filterExpression, sorting, pagination, requestScope) -> - tx.getRelation(relationTx, entity, relationship.copyOf() + tx.getToManyRelation(relationTx, entity, relationship.copyOf() .projection(relationship.getProjection().copyOf() .filterExpression(filterExpression.orElse(null)) .sorting(sorting.orElse(null)) @@ -95,23 +95,22 @@ public Object getRelation(DataStoreTransaction relationTx, * It must be done in memory by Elide as some newly created entities have not yet been persisted. */ boolean filterInMemory = scope.getNewPersistentResources().size() > 0; - return fetchData(fetcher, Optional.of(entity), relationship.getProjection(), filterInMemory, scope); + return fetchData(fetcher, relationship.getProjection(), filterInMemory, scope); } @Override public Object loadObject(EntityProjection projection, Serializable id, RequestScope scope) { - - if (projection.getFilterExpression() == null - || tx.supportsFiltering(scope, Optional.empty(), projection) == FeatureSupport.FULL) { + if (projection.getFilterExpression() == null) { return tx.loadObject(projection, id, scope); } + return DataStoreTransaction.super.loadObject(projection, id, scope); } @Override - public Iterable loadObjects(EntityProjection projection, + public DataStoreIterable loadObjects(EntityProjection projection, RequestScope scope) { DataFetcher fetcher = (filterExpression, sorting, pagination, requestScope) -> @@ -121,7 +120,7 @@ public Iterable loadObjects(EntityProjection projection, .sorting(sorting.orElse(null)) .build(), requestScope); - return (Iterable) fetchData(fetcher, Optional.empty(), projection, false, scope); + return fetchData(fetcher, projection, false, scope); } @Override @@ -144,6 +143,15 @@ public T createNewObject(Type entityClass, RequestScope scope) { return tx.createNewObject(entityClass, scope); } + @Override + public R getToOneRelation( + DataStoreTransaction relationTx, + T entity, Relationship relationship, + RequestScope scope + ) { + return tx.getToOneRelation(relationTx, entity, relationship, scope); + } + @Override public void close() throws IOException { tx.close(); @@ -193,85 +201,104 @@ public void createObject(Object entity, RequestScope scope) { tx.createObject(entity, scope); } - private Iterable filterLoadedData(Iterable loadedRecords, + private DataStoreIterable filterLoadedData(DataStoreIterable loadedRecords, Optional filterExpression, RequestScope scope) { + if (! filterExpression.isPresent()) { return loadedRecords; } - Predicate predicate = filterExpression.get().accept(new InMemoryFilterExecutor(scope)); + return new DataStoreIterable<>() { + @Override + public Iterable getWrappedIterable() { + return loadedRecords; + } + + @Override + public Iterator iterator() { + return new FilteredIterator<>(filterExpression.get(), scope, loadedRecords.iterator()); + } - return StreamSupport.stream(loadedRecords.spliterator(), false) - .filter(predicate::test) - .collect(Collectors.toList()); - } + @Override + public boolean needsInMemoryFilter() { + return true; + } + + @Override + public boolean needsInMemorySort() { + return true; + } - private Object fetchData(DataFetcher fetcher, - Optional parent, - EntityProjection projection, - boolean filterInMemory, - RequestScope scope) { + @Override + public boolean needsInMemoryPagination() { + return true; + } + }; + } + private DataStoreIterable fetchData( + DataFetcher fetcher, + EntityProjection projection, + boolean filterInMemory, + RequestScope scope + ) { Optional filterExpression = Optional.ofNullable(projection.getFilterExpression()); Pair, Optional> expressionSplit = splitFilterExpression( - scope, parent, projection, filterInMemory); + scope, projection, filterInMemory); Optional dataStoreFilter = expressionSplit.getLeft(); Optional inMemoryFilter = expressionSplit.getRight(); - Pair, Optional> sortSplit = splitSorting(scope, parent, - projection, inMemoryFilter.isPresent()); - - Optional dataStoreSort = sortSplit.getLeft(); - Optional inMemorySort = sortSplit.getRight(); + Optional dataStoreSorting = getDataStoreSorting(scope, projection, filterInMemory); - Pair, Optional> paginationSplit = splitPagination(scope, parent, - projection, inMemoryFilter.isPresent(), inMemorySort.isPresent()); + boolean sortingInMemory = dataStoreSorting.isEmpty() && projection.getSorting() != null; - Optional dataStorePagination = paginationSplit.getLeft(); - Optional inMemoryPagination = paginationSplit.getRight(); + Optional dataStorePagination = inMemoryFilter.isPresent() || sortingInMemory + ? Optional.empty() : Optional.ofNullable(projection.getPagination()); - Object result = fetcher.fetch(dataStoreFilter, dataStoreSort, dataStorePagination, scope); + DataStoreIterable loadedRecords = + fetcher.fetch(dataStoreFilter, dataStoreSorting, dataStorePagination, scope); - if (! (result instanceof Iterable)) { - return result; + if (loadedRecords == null) { + return new DataStoreIterableBuilder().build(); } - Iterable loadedRecords = (Iterable) result; - - if (inMemoryFilter.isPresent()) { + if (inMemoryFilter.isPresent() || (loadedRecords.needsInMemoryFilter() + && projection.getFilterExpression() != null)) { loadedRecords = filterLoadedData(loadedRecords, filterExpression, scope); } - return sortAndPaginateLoadedData( loadedRecords, - inMemorySort, - inMemoryPagination, + sortingInMemory, + projection.getSorting(), + projection.getPagination(), scope); } + private DataStoreIterable sortAndPaginateLoadedData( + DataStoreIterable loadedRecords, + boolean sortingInMemory, + Sorting sorting, + Pagination pagination, + RequestScope scope + ) { + + Map sortRules = sorting == null ? new HashMap<>() : sorting.getSortingPaths(); + + boolean mustSortInMemory = ! sortRules.isEmpty() + && (sortingInMemory || loadedRecords.needsInMemorySort()); - private Iterable sortAndPaginateLoadedData(Iterable loadedRecords, - Optional sorting, - Optional pagination, - RequestScope scope) { + boolean mustPaginateInMemory = pagination != null + && (mustSortInMemory || loadedRecords.needsInMemoryPagination()); //Try to skip the data copy if possible - if (! sorting.isPresent() && ! pagination.isPresent()) { + if (! mustSortInMemory && ! mustPaginateInMemory) { return loadedRecords; } - Map sortRules = sorting - .map(Sorting::getSortingPaths) - .orElseGet(HashMap::new); - - // No sorting required for this type & no pagination. - if (sortRules.isEmpty() && ! pagination.isPresent()) { - return loadedRecords; - } //We need an in memory copy to sort or paginate. List results = StreamSupport.stream(loadedRecords.spliterator(), false).collect(Collectors.toList()); @@ -279,11 +306,11 @@ private Iterable sortAndPaginateLoadedData(Iterable loadedRecord results = sortInMemory(results, sortRules, scope); } - if (pagination.isPresent()) { - results = paginateInMemory(results, pagination.get()); + if (pagination != null) { + results = paginateInMemory(results, pagination); } - return results; + return new DataStoreIterableBuilder(results).build(); } private List paginateInMemory(List records, Pagination pagination) { @@ -345,18 +372,51 @@ private Comparator getComparator(Path path, Sorting.SortOrder order, Req } /** - * Splits a filter expression into two components: + * Returns the sorting (if any) that should be pushed to the datastore. + * @param scope The request context + * @param projection The projection being loaded. + * @param filterInMemory Whether or not the transaction requires in memory filtering. + * @return An optional sorting. + */ + private Optional getDataStoreSorting( + RequestScope scope, + EntityProjection projection, + boolean filterInMemory + ) { + Sorting sorting = projection.getSorting(); + if (filterInMemory) { + return Optional.empty(); + } + Map sortRules = sorting == null ? new HashMap<>() : sorting.getSortingPaths(); + + boolean sortingOnComputedAttribute = false; + for (Path path: sortRules.keySet()) { + if (path.isComputed(scope.getDictionary())) { + Type pathType = path.getPathElements().get(0).getType(); + if (projection.getType().equals(pathType)) { + sortingOnComputedAttribute = true; + break; + } + } + } + if (sortingOnComputedAttribute) { + return Optional.empty(); + } else { + return Optional.ofNullable(sorting); + } + } + + /** + * Splits a filter expression into two components. They are: * - a component that should be pushed down to the data store * - a component that should be executed in memory * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. * @param projection The projection being loaded. * @param filterInMemory Whether or not the transaction requires in memory filtering. * @return A pair of filter expressions (data store expression, in memory expression) */ private Pair, Optional> splitFilterExpression( RequestScope scope, - Optional parent, EntityProjection projection, boolean filterInMemory ) { @@ -368,11 +428,7 @@ private Pair, Optional> splitFilter boolean transactionNeedsInMemoryFiltering = filterInMemory; if (filterExpression.isPresent()) { - FeatureSupport filterSupport = tx.supportsFiltering(scope, parent, projection); - - boolean storeNeedsInMemoryFiltering = filterSupport != FeatureSupport.FULL; - - if (transactionNeedsInMemoryFiltering || filterSupport == FeatureSupport.NONE) { + if (transactionNeedsInMemoryFiltering) { inStoreFilterExpression = Optional.empty(); } else { inStoreFilterExpression = Optional.ofNullable( @@ -383,7 +439,7 @@ private Pair, Optional> splitFilter boolean expressionNeedsInMemoryFiltering = InMemoryExecutionVerifier.shouldExecuteInMemory( scope.getDictionary(), filterExpression.get()); - if (transactionNeedsInMemoryFiltering || storeNeedsInMemoryFiltering || expressionNeedsInMemoryFiltering) { + if (transactionNeedsInMemoryFiltering || expressionNeedsInMemoryFiltering) { inMemoryFilterExpression = filterExpression; } } @@ -391,68 +447,6 @@ private Pair, Optional> splitFilter return Pair.of(inStoreFilterExpression, inMemoryFilterExpression); } - /** - * Splits a sorting object into two components: - * - a component that should be pushed down to the data store - * - a component that should be executed in memory - * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. - * @param projection The projection being loaded. - * @param filteredInMemory Whether or not filtering was performed in memory - * @return A pair of sorting objects (data store sort, in memory sort) - */ - private Pair, Optional> splitSorting( - RequestScope scope, - Optional parent, - EntityProjection projection, - boolean filteredInMemory - ) { - Optional sorting = Optional.ofNullable(projection.getSorting()); - - if (sorting.isPresent() && (! tx.supportsSorting(scope, parent, projection) || filteredInMemory)) { - return Pair.of(Optional.empty(), sorting); - } - return Pair.of(sorting, Optional.empty()); - } - - /** - * Splits a pagination object into two components: - * - a component that should be pushed down to the data store - * - a component that should be executed in memory - * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. - * @param projection The projection being loaded. - * @param filteredInMemory Whether or not filtering was performed in memory - * @param sortedInMemory Whether or not sorting was performed in memory - * @return A pair of pagination objects (data store pagination, in memory pagination) - */ - private Pair, Optional> splitPagination( - RequestScope scope, - Optional parent, - EntityProjection projection, - boolean filteredInMemory, - boolean sortedInMemory - ) { - - Optional pagination = Optional.ofNullable(projection.getPagination()); - - if (!tx.supportsPagination(scope, parent, projection) - || filteredInMemory - || sortedInMemory) { - return Pair.of(Optional.empty(), pagination); - - /* - * For default pagination, we let the store do its work, but we also let the store ignore pagination - * by also performing in memory. This allows the ORM the opportunity to manage its own SQL query generation - * to avoid N+1. - */ - } else if (pagination.isPresent() && pagination.get().isDefaultInstance()) { - return Pair.of(pagination, pagination); - } else { - return Pair.of(pagination, Optional.empty()); - } - } - @Override public void cancel(RequestScope scope) { tx.cancel(scope); diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java index 220757e617..dd9781ef8a 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java @@ -7,6 +7,7 @@ package com.yahoo.elide.core.datastore.wrapped; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; @@ -16,7 +17,6 @@ import java.io.IOException; import java.io.Serializable; -import java.util.Optional; import java.util.Set; /** @@ -39,9 +39,15 @@ public T loadObject(EntityProjection projection, Serializable id, } @Override - public R getRelation(DataStoreTransaction relationTx, T entity, - Relationship relationship, RequestScope scope) { - return tx.getRelation(relationTx, entity, relationship, scope); + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, T entity, + Relationship relationship, RequestScope scope) { + return tx.getToManyRelation(relationTx, entity, relationship, scope); + } + + @Override + public R getToOneRelation(DataStoreTransaction relationTx, T entity, + Relationship relationship, RequestScope scope) { + return tx.getToOneRelation(relationTx, entity, relationship, scope); } @Override @@ -68,21 +74,6 @@ public void setAttribute(T entity, Attribute attribute, RequestScope scope) tx.setAttribute(entity, attribute, scope); } - @Override - public FeatureSupport supportsFiltering(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsFiltering(scope, parent, projection); - } - - @Override - public boolean supportsSorting(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsSorting(scope, parent, projection); - } - - @Override - public boolean supportsPagination(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsPagination(scope, parent, projection); - } - @Override public void save(T o, RequestScope requestScope) { tx.save(o, requestScope); @@ -109,7 +100,7 @@ public void createObject(Object o, RequestScope requestScope) { } @Override - public Iterable loadObjects(EntityProjection projection, RequestScope scope) { + public DataStoreIterable loadObjects(EntityProjection projection, RequestScope scope) { return tx.loadObjects(projection, scope); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java index 29ae71c304..7024a6a66d 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java @@ -6,6 +6,7 @@ package com.yahoo.elide.core.dictionary; import com.yahoo.elide.core.type.Type; +import lombok.Builder; import lombok.Value; /** @@ -21,6 +22,7 @@ public ArgumentType(String name, Type type) { this(name, type, null); } + @Builder public ArgumentType(String name, Type type, Object defaultValue) { this.name = name; this.type = type; diff --git a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java index d3524d38b4..863e4fbe97 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java @@ -67,6 +67,8 @@ * @see com.yahoo.elide.annotation.Include#name */ public class EntityBinding { + public static final List
- * This method does not handle the case where the toAdd entity is newly created, - * since newly created entities have null id there is no easy way to check for identity - */ - public boolean relationshipAlreadyExists(String fieldName, PersistentResource toAdd) { - Object relation = this.getValueUnchecked(fieldName); - String toAddId = toAdd.getId(); - if (toAddId == null) { - return false; - } - if (relation instanceof Collection) { - return ((Collection) relation).stream().anyMatch(obj -> toAddId.equals(dictionary.getId(obj))); - } - return toAddId.equals(dictionary.getId(relation)); - } - /** * Add relation link from a given parent resource to a child resource. * @@ -958,21 +975,20 @@ public boolean relationshipAlreadyExists(String fieldName, PersistentResource to * @param newRelation the new relation */ public void addRelation(String fieldName, PersistentResource newRelation) { - if (!newRelation.isNewlyCreated() && relationshipAlreadyExists(fieldName, newRelation)) { - return; - } checkTransferablePermission(Collections.singleton(newRelation)); Object relation = this.getValueUnchecked(fieldName); if (relation instanceof Collection) { - if (addToCollection((Collection) relation, fieldName, newRelation)) { + if (modifyCollection((Collection) relation, fieldName, + Set.of(newRelation.getObject()), Collections.emptySet(), true)) { this.markDirty(); - } - //Hook for updateToManyRelation - transaction.updateToManyRelation(transaction, obj, fieldName, - Sets.newHashSet(newRelation.getObject()), new LinkedHashSet<>(), requestScope); - addInverseRelation(fieldName, newRelation.getObject()); + //Hook for updateToManyRelation + transaction.updateToManyRelation(transaction, obj, fieldName, + Sets.newHashSet(newRelation.getObject()), new LinkedHashSet<>(), requestScope); + + addInverseRelation(fieldName, newRelation.getObject()); + } } else { // Not a collection, but may be trying to create a ToOne relationship. // NOTE: updateRelation marks dirty. @@ -1221,12 +1237,6 @@ private void assertPropertyExists(String propertyName) { private Observable getRelation(com.yahoo.elide.core.request.Relationship relationship, boolean checked) { - if (checked) { - //All getRelation calls funnel to here. We only publish events for actions triggered directly - //by the API client. - requestScope.publishLifecycleEvent(this, READ); - requestScope.publishLifecycleEvent(this, relationship.getName(), READ, Optional.empty()); - } if (checked && !checkRelation(relationship)) { return Observable.empty(); @@ -1324,19 +1334,21 @@ private Observable getRelationUnchecked( .build() ).build(); - Object val = transaction.getRelation(transaction, obj, modifiedRelationship, requestScope); - - if (val == null) { - return Observable.empty(); - } - Observable resources; - if (val instanceof Iterable) { - Iterable filteredVal = (Iterable) val; + if (type.isToMany()) { + DataStoreIterable val = transaction.getToManyRelation(transaction, obj, modifiedRelationship, requestScope); + + if (val == null) { + return Observable.empty(); + } resources = Observable.fromIterable( - new PersistentResourceSet(this, relationName, filteredVal, requestScope)); + new PersistentResourceSet(this, relationName, val, requestScope)); } else { + Object val = transaction.getToOneRelation(transaction, obj, modifiedRelationship, requestScope); + if (val == null) { + return Observable.empty(); + } resources = Observable.fromArray(new PersistentResource(val, this, relationName, requestScope.getUUIDFor(val), requestScope)); } @@ -1409,7 +1421,7 @@ public void setObject(T obj) { @Override @JsonIgnore public Type getResourceType() { - return (Type) dictionary.lookupBoundClass(EntityDictionary.getType(obj)); + return (Type) dictionary.lookupBoundClass(getType(obj)); } /** @@ -1544,6 +1556,26 @@ public Resource toResource(final Map relationships, if (requestScope.getElideSettings().isEnableJsonLinks()) { resource.setLinks(requestScope.getElideSettings().getJsonApiLinks().getResourceLevelLinks(this)); } + + if (! (getObject() instanceof WithMetadata)) { + return resource; + } + + WithMetadata withMetadata = (WithMetadata) getObject(); + Set fields = withMetadata.getMetadataFields(); + + if (fields.size() == 0) { + return resource; + } + + Meta meta = new Meta(new HashMap<>()); + + for (String field : fields) { + meta.getMetaMap().put(field, withMetadata.getMetadataField(field).get()); + } + + resource.setMeta(meta); + return resource; } @@ -1677,8 +1709,6 @@ protected void nullValue(String fieldName, PersistentResource oldValue) { * @return value value */ protected Object getValueChecked(Attribute attribute) { - requestScope.publishLifecycleEvent(this, READ); - requestScope.publishLifecycleEvent(this, attribute.getName(), READ, Optional.empty()); checkFieldAwareDeferPermissions(ReadPermission.class, attribute.getName(), null, null); return transaction.getAttribute(getObject(), attribute, requestScope); } @@ -1693,94 +1723,47 @@ protected Object getValueUnchecked(String fieldName) { return getValue(getObject(), fieldName, requestScope); } - /** - * Adds a new element to a collection and tests update permission. - * - * @param collection the collection - * @param collectionName the collection name - * @param toAdd the to add - * @return True if added to collection false otherwise (i.e. element already in collection) - */ - protected boolean addToCollection(Collection collection, String collectionName, PersistentResource toAdd) { - final Collection singleton = Collections.singleton(toAdd.getObject()); - final Collection original = copyCollection(collection); - checkFieldAwareDeferPermissions( - UpdatePermission.class, - collectionName, - CollectionUtils.union(CollectionUtils.emptyIfNull(collection), singleton), - original); - if (collection == null) { - collection = Collections.singleton(toAdd.getObject()); - Object value = getValueUnchecked(collectionName); - if (!Objects.equals(value, toAdd.getObject())) { - this.setValueChecked(collectionName, collection); - return true; - } - } else { - if (!collection.contains(toAdd.getObject())) { - collection.add(toAdd.getObject()); + protected boolean modifyCollection( + Collection toModify, + String collectionName, + Collection toAdd, + Collection toRemove, + boolean updateInverse) { - triggerUpdate(collectionName, original, collection); - return true; - } - } - return false; - } + Collection copyOfOriginal = copyCollection(toModify); + + Collection modified = CollectionUtils.union(CollectionUtils.emptyIfNull(toModify), toAdd); + modified = CollectionUtils.subtract(modified, toRemove); - /** - * Deletes an existing element in a collection and tests update and delete permissions. - * - * @param collection the collection - * @param collectionName the collection name - * @param toDelete the to delete - * @param isInverseCheck Whether or not the deletion is already coming from cleaning up an inverse. - * Without this parameter, we could find ourselves in a loop of checks. - * TODO: This is a band-aid for a quick fix. This should certainly be refactored. - */ - protected void delFromCollection( - Collection collection, - String collectionName, - PersistentResource toDelete, - boolean isInverseCheck) { - final Collection original = copyCollection(collection); checkFieldAwareDeferPermissions( UpdatePermission.class, collectionName, - CollectionUtils.disjunction(collection, Collections.singleton(toDelete.getObject())), - original - ); - - String inverseField = getInverseRelationField(collectionName); - if (!isInverseCheck && !inverseField.isEmpty()) { - // Compute the ChangeSpec for the inverse relation and check whether or not we have access - // to apply this change to that field. - final Object originalValue = toDelete.getValueUnchecked(inverseField); - final Collection originalBidirectional; + modified, + copyOfOriginal); - if (originalValue instanceof Collection) { - originalBidirectional = copyCollection((Collection) originalValue); - } else { - originalBidirectional = Collections.singleton(originalValue); + if (updateInverse) { + for (Object adding : toAdd) { + addInverseRelation(collectionName, adding); } - final Collection removedBidrectional = CollectionUtils - .disjunction(Collections.singleton(this.getObject()), originalBidirectional); - - toDelete.checkFieldAwareDeferPermissions( - UpdatePermission.class, - inverseField, - removedBidrectional, - originalBidirectional - ); - } - - if (collection == null) { - return; + for (Object removing : toRemove) { + deleteInverseRelation(collectionName, removing); + } } - collection.remove(toDelete.getObject()); + if (toModify == null) { + this.setValueChecked(collectionName, modified); + return true; + } else { + if (copyOfOriginal.equals(modified)) { + return false; + } + toModify.addAll(toAdd); + toModify.removeAll(toRemove); - triggerUpdate(collectionName, original, collection); + triggerUpdate(collectionName, copyOfOriginal, modified); + return true; + } } /** @@ -1820,7 +1803,8 @@ protected void deleteInverseRelation(String relationName, Object inverseEntity) } if (inverseRelation instanceof Collection) { - inverseResource.delFromCollection((Collection) inverseRelation, inverseField, this, true); + inverseResource.modifyCollection((Collection) inverseRelation, inverseField, + Collections.emptySet(), Set.of(this.getObject()), false); } else if (inverseType.isAssignableFrom(this.getResourceType())) { inverseResource.nullValue(inverseField, this); } else { @@ -1870,7 +1854,8 @@ protected void addInverseRelation(String relationName, Object inverseObj) { if (COLLECTION_TYPE.isAssignableFrom(inverseType)) { if (inverseRelation != null) { - inverseResource.addToCollection((Collection) inverseRelation, inverseName, this); + inverseResource.modifyCollection((Collection) inverseRelation, inverseName, + Set.of(this.getObject()), Collections.emptySet(), false); } else { inverseResource.setValueChecked(inverseName, Collections.singleton(this.getObject())); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java index c1ccdb8213..c32ae0854c 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/RequestScope.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.core; +import static com.yahoo.elide.annotation.LifeCycleHookBinding.ALL_OPERATIONS; import com.yahoo.elide.ElideSettings; import com.yahoo.elide.annotation.LifeCycleHookBinding; import com.yahoo.elide.core.audit.AuditLogger; @@ -27,9 +28,6 @@ import com.yahoo.elide.jsonapi.JsonApiMapper; import com.yahoo.elide.jsonapi.models.JsonApiDocument; import org.apache.commons.collections.MapUtils; -import io.reactivex.Observable; -import io.reactivex.subjects.PublishSubject; -import io.reactivex.subjects.ReplaySubject; import lombok.Getter; import lombok.Setter; @@ -76,9 +74,9 @@ public class RequestScope implements com.yahoo.elide.core.security.RequestScope @Getter private final UUID requestId; private final Map expressionsByType; - private PublishSubject lifecycleEvents; - private Observable distinctLifecycleEvents; - private ReplaySubject queuedLifecycleEvents; + private final Map metadata; + + private LinkedHashSet eventQueue; /* Used to filter across heterogeneous types during the first load */ private FilterExpression globalFilterExpression; @@ -107,10 +105,7 @@ public RequestScope(String baseUrlEndPoint, UUID requestId, ElideSettings elideSettings) { this.apiVersion = apiVersion; - this.lifecycleEvents = PublishSubject.create(); - this.distinctLifecycleEvents = lifecycleEvents.distinct(); - this.queuedLifecycleEvents = ReplaySubject.create(); - this.distinctLifecycleEvents.subscribe(queuedLifecycleEvents); + this.eventQueue = new LinkedHashSet<>(); this.path = path; this.baseUrlEndPoint = baseUrlEndPoint; @@ -132,12 +127,12 @@ public RequestScope(String baseUrlEndPoint, this.dirtyResources = new LinkedHashSet<>(); this.deletedResources = new LinkedHashSet<>(); this.requestId = requestId; + this.metadata = new HashMap<>(); this.queryParams = queryParams == null ? new MultivaluedHashMap<>() : queryParams; this.requestHeaders = MapUtils.isEmpty(requestHeaders) ? Collections.emptyMap() : requestHeaders; - registerPreSecurityObservers(); this.sparseFields = parseSparseFields(getQueryParams()); @@ -218,12 +213,11 @@ protected RequestScope(String path, String apiVersion, this.filterDialect = outerRequestScope.filterDialect; this.expressionsByType = outerRequestScope.expressionsByType; this.elideSettings = outerRequestScope.elideSettings; - this.lifecycleEvents = outerRequestScope.lifecycleEvents; - this.distinctLifecycleEvents = outerRequestScope.distinctLifecycleEvents; + this.eventQueue = outerRequestScope.eventQueue; this.updateStatusCode = outerRequestScope.updateStatusCode; - this.queuedLifecycleEvents = outerRequestScope.queuedLifecycleEvents; this.requestId = outerRequestScope.requestId; this.sparseFields = outerRequestScope.sparseFields; + this.metadata = new HashMap<>(outerRequestScope.metadata); } public Set getNewResources() { @@ -326,115 +320,59 @@ private static MultivaluedMap getFilterParams(MultivaluedMap event.getEventType().equals(operation)) + .forEach(event -> { + invoker.onNext(event); + }); + } + /** * Run queued pre-security lifecycle triggers. */ public void runQueuedPreSecurityTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, false)) - .throwOnError(); + notifySubscribers(LifeCycleHookBinding.Operation.CREATE, LifeCycleHookBinding.TransactionPhase.PRESECURITY); } /** * Run queued pre-flush lifecycle triggers. */ public void runQueuedPreFlushTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PREFLUSH, false)) - .throwOnError(); + runQueuedPreFlushTriggers(ALL_OPERATIONS); + } + + /** + * Run queued pre-flush lifecycle triggers. + * @param operations List of operations to run pre-flush triggers for. + */ + public void runQueuedPreFlushTriggers(LifeCycleHookBinding.Operation[] operations) { + for (LifeCycleHookBinding.Operation op : operations) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.PREFLUSH); + } } /** * Run queued pre-commit lifecycle triggers. */ public void runQueuedPreCommitTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PRECOMMIT, false)) - .throwOnError(); + for (LifeCycleHookBinding.Operation op : ALL_OPERATIONS) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.PRECOMMIT); + } } /** * Run queued post-commit lifecycle triggers. */ public void runQueuedPostCommitTriggers() { - this.queuedLifecycleEvents - .filter(CRUDEvent::isCreateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.CREATE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); - - this.queuedLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.POSTCOMMIT, false)) - .throwOnError(); + for (LifeCycleHookBinding.Operation op : ALL_OPERATIONS) { + notifySubscribers(op, LifeCycleHookBinding.TransactionPhase.POSTCOMMIT); + } } /** @@ -444,9 +382,7 @@ public void runQueuedPostCommitTriggers() { * @param crudAction CRUD action */ protected void publishLifecycleEvent(PersistentResource resource, LifeCycleHookBinding.Operation crudAction) { - lifecycleEvents.onNext( - new CRUDEvent(crudAction, resource, PersistentResource.CLASS_NO_FIELD, Optional.empty()) - ); + publishLifecycleEvent(new CRUDEvent(crudAction, resource, PersistentResource.CLASS_NO_FIELD, Optional.empty())); } /** @@ -461,9 +397,23 @@ protected void publishLifecycleEvent(PersistentResource resource, String fieldName, LifeCycleHookBinding.Operation crudAction, Optional changeSpec) { - lifecycleEvents.onNext( - new CRUDEvent(crudAction, resource, fieldName, changeSpec) - ); + publishLifecycleEvent(new CRUDEvent(crudAction, resource, fieldName, changeSpec)); + } + + protected void publishLifecycleEvent(CRUDEvent event) { + if (! eventQueue.contains(event)) { + if (event.getEventType().equals(LifeCycleHookBinding.Operation.DELETE) + || event.getEventType().equals(LifeCycleHookBinding.Operation.UPDATE)) { + + LifecycleHookInvoker invoker = new LifecycleHookInvoker(dictionary, + event.getEventType(), + LifeCycleHookBinding.TransactionPhase.PRESECURITY); + + invoker.onNext(event); + } + + eventQueue.add(event); + } } public void saveOrCreateObjects() { @@ -511,27 +461,6 @@ private String getInheritanceKey(String subClass, String superClass) { return subClass + "!" + superClass; } - private void registerPreSecurityObservers() { - - this.distinctLifecycleEvents - .filter(CRUDEvent::isReadEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.READ, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - - this.distinctLifecycleEvents - .filter(CRUDEvent::isUpdateEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.UPDATE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - - this.distinctLifecycleEvents - .filter(CRUDEvent::isDeleteEvent) - .subscribeWith(new LifecycleHookInvoker(dictionary, - LifeCycleHookBinding.Operation.DELETE, - LifeCycleHookBinding.TransactionPhase.PRESECURITY, true)); - } - @Override public String getRequestHeaderByName(String headerName) { if (this.requestHeaders.get(headerName) == null) { @@ -539,4 +468,19 @@ public String getRequestHeaderByName(String headerName) { } return this.requestHeaders.get(headerName).get(0); } + + @Override + public void setMetadataField(String property, Object value) { + metadata.put(property, value); + } + + @Override + public Optional getMetadataField(String property) { + return Optional.ofNullable(metadata.getOrDefault(property, null)); + } + + @Override + public Set getMetadataFields() { + return metadata.keySet(); + } } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java b/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java index 291bb477a4..ab279214e7 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/TransactionRegistry.java @@ -8,15 +8,16 @@ import com.yahoo.elide.core.datastore.DataStoreTransaction; import lombok.Getter; -import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + /** * Transaction Registry class. */ @Getter public class TransactionRegistry { - private Map transactionMap = new HashMap<>(); + private Map transactionMap = new ConcurrentHashMap<>(); public Map getRunningTransactions() { return transactionMap; diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java new file mode 100644 index 0000000000..88c647dfd3 --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterable.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.core.datastore; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Consumer; + +/** + * Returns data loaded from a DataStore. Wraps an iterable but also communicates to Elide + * if the framework needs to filter, sort, or paginate the iterable in memory before returning to the client. + * @param The type being iterated over. + */ +public interface DataStoreIterable extends Iterable { + + /** + * Returns the underlying iterable. + * @return The underlying iterable. + */ + Iterable getWrappedIterable(); + + @Override + default Iterator iterator() { + return getWrappedIterable().iterator(); + } + + @Override + default void forEach(Consumer action) { + getWrappedIterable().forEach(action); + } + + @Override + default Spliterator spliterator() { + return Spliterators.spliteratorUnknownSize(iterator(), 0); + } + + /** + * Whether the iterable should be filtered in memory. + * @return true if the iterable needs sorting in memory. false otherwise. + */ + default boolean needsInMemoryFilter() { + return false; + } + + /** + * Whether the iterable should be sorted in memory. + * @return true if the iterable needs sorting in memory. false otherwise. + */ + default boolean needsInMemorySort() { + return false; + } + + /** + * Whether the iterable should be paginated in memory. + * @return true if the iterable needs pagination in memory. false otherwise. + */ + default boolean needsInMemoryPagination() { + return false; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java new file mode 100644 index 0000000000..ba12743df0 --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreIterableBuilder.java @@ -0,0 +1,109 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.core.datastore; + +import java.util.ArrayList; + +/** + * Constructs DataStoreIterables. + * @param + */ +public class DataStoreIterableBuilder { + + private boolean filterInMemory = false; + private boolean sortInMemory = false; + private boolean paginateInMemory = false; + private final Iterable wrapped; + + /** + * Constructor. + */ + public DataStoreIterableBuilder() { + this.wrapped = new ArrayList<>(); + } + + /** + * Constructor. + * @param wrapped Required iterable to wrap. + */ + public DataStoreIterableBuilder(Iterable wrapped) { + if (wrapped == null) { + this.wrapped = new ArrayList<>(); + } else { + this.wrapped = wrapped; + } + } + + /** + * Filter the iterable in memory. + * @param filterInMemory true to filter in memory. + * @return the builder. + */ + public DataStoreIterableBuilder filterInMemory(boolean filterInMemory) { + this.filterInMemory = filterInMemory; + return this; + } + + /** + * Sorts the iterable in memory. + * @param sortInMemory true to sort in memory. + * @return the builder. + */ + public DataStoreIterableBuilder sortInMemory(boolean sortInMemory) { + this.sortInMemory = sortInMemory; + return this; + } + + /** + * Paginates the iterable in memory. + * @param paginateInMemory true to paginate in memory. + * @return the builder. + */ + public DataStoreIterableBuilder paginateInMemory(boolean paginateInMemory) { + this.paginateInMemory = paginateInMemory; + return this; + } + + /** + * Filter, sort, and paginate in memory. + * @return the builder. + */ + public DataStoreIterableBuilder allInMemory() { + this.filterInMemory = true; + this.sortInMemory = true; + this.paginateInMemory = true; + return this; + } + + /** + * Constructs the DataStoreIterable. + * @return the new iterable. + */ + public DataStoreIterable build() { + return new DataStoreIterable() { + @Override + public Iterable getWrappedIterable() { + return wrapped; + } + + @Override + public boolean needsInMemoryFilter() { + return filterInMemory; + } + + @Override + public boolean needsInMemorySort() { + return sortInMemory; + } + + @Override + public boolean needsInMemoryPagination() { + return paginateInMemory; + } + }; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java index b88dd0c3dc..22c5da62d3 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/DataStoreTransaction.java @@ -23,21 +23,11 @@ import java.io.Closeable; import java.io.Serializable; import java.util.Iterator; -import java.util.Optional; import java.util.Set; /** * Wraps the Database Transaction type. */ public interface DataStoreTransaction extends Closeable { - - /** - * The extent to which the transaction supports a particular feature. - */ - public enum FeatureSupport { - FULL, - PARTIAL, - NONE - } /** * Save the updated object. * @@ -170,12 +160,12 @@ default T loadObject(EntityProjection entityProjection, * @param - The model type being loaded. * @return a collection of the loaded objects */ - Iterable loadObjects( + DataStoreIterable loadObjects( EntityProjection entityProjection, RequestScope scope); /** - * Retrieve a relation from an object. + * Retrieve a to-many relation from an object. * * @param relationTx - The datastore that governs objects of the relationhip's type. * @param entity - The object which owns the relationship. @@ -185,7 +175,28 @@ Iterable loadObjects( * @param - The model type of the relationship. * @return the object in the relation */ - default R getRelation( + default DataStoreIterable getToManyRelation( + DataStoreTransaction relationTx, + T entity, + Relationship relationship, + RequestScope scope) { + + return new DataStoreIterableBuilder( + (Iterable) PersistentResource.getValue(entity, relationship.getName(), scope)).allInMemory().build(); + } + + /** + * Retrieve a to-one relation from an object. + * + * @param relationTx - The datastore that governs objects of the relationhip's type. + * @param entity - The object which owns the relationship. + * @param relationship - the relationship to fetch. + * @param scope - contains request level metadata. + * @param - The model type which owns the relationship. + * @param - The model type of the relationship. + * @return the object in the relation + */ + default R getToOneRelation( DataStoreTransaction relationTx, T entity, Relationship relationship, @@ -270,48 +281,6 @@ default void setAttribute(T entity, RequestScope scope) { } - /** - * Whether or not the transaction can filter the provided class with the provided expression. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being filtered). - * @return FULL, PARTIAL, or NONE - */ - default FeatureSupport supportsFiltering(RequestScope scope, - Optional parent, - EntityProjection projection) { - return FeatureSupport.FULL; - } - - /** - * Whether or not the transaction can sort the provided class. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being sorted). - * @return true if sorting is possible - */ - default boolean supportsSorting(RequestScope scope, - Optional parent, - EntityProjection projection) { - return true; - } - - /** - * Whether or not the transaction can paginate the provided class. - * @param scope The request scope - * @param projection The projection being loaded - * @param parent Are we filtering a root collection or a relationship - * @param - The model type of the parent model (if a relationship is being paginated). - * @return true if pagination is possible - */ - default boolean supportsPagination(RequestScope scope, - Optional parent, - EntityProjection projection) { - return true; - } - /** * Cancel running transaction. * Implementation must be thread-safe. diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java new file mode 100644 index 0000000000..d8d479e95b --- /dev/null +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/FilteredIterator.java @@ -0,0 +1,77 @@ +/* + * Copyright 2021, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.core.datastore.inmemory; + +import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.filter.expression.FilterExpression; +import com.yahoo.elide.core.filter.expression.InMemoryFilterExecutor; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.function.Predicate; + +/** + * An iterator which filters another iterator by an Elide filter expression. + * @param The type being iterated over. + */ +public class FilteredIterator implements Iterator { + + private Iterator wrapped; + private Predicate predicate; + + private T next; + + /** + * Constructor. + * @param filterExpression The filter expression to filter on. + * @param scope Request scope. + * @param wrapped The wrapped iterator. + */ + public FilteredIterator(FilterExpression filterExpression, RequestScope scope, Iterator wrapped) { + this.wrapped = wrapped; + InMemoryFilterExecutor executor = new InMemoryFilterExecutor(scope); + + predicate = filterExpression.accept(executor); + } + + @Override + public boolean hasNext() { + try { + next = next(); + } catch (NoSuchElementException e) { + return false; + } + + return true; + } + + @Override + public T next() { + if (next != null) { + T result = next; + next = null; + return result; + } + + while (next == null && wrapped.hasNext()) { + try { + next = wrapped.next(); + } catch (NoSuchElementException e) { + next = null; + } + if (next == null || ! predicate.test(next)) { + next = null; + } + } + + if (next == null) { + throw new NoSuchElementException(); + } + + return next; + } +} diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java index a3520b5d00..45eee754e4 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/HashMapStoreTransaction.java @@ -6,6 +6,8 @@ package com.yahoo.elide.core.datastore.inmemory; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; +import com.yahoo.elide.core.datastore.DataStoreIterableBuilder; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.dictionary.EntityDictionary; import com.yahoo.elide.core.exceptions.TransactionException; @@ -19,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import javax.persistence.GeneratedValue; @@ -125,19 +126,20 @@ public void setId(Object value, String id) { } @Override - public Object getRelation(DataStoreTransaction relationTx, - Object entity, - Relationship relationship, - RequestScope scope) { - return dictionary.getValue(entity, relationship.getName(), scope); + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, + Object entity, + Relationship relationship, + RequestScope scope) { + return new DataStoreIterableBuilder( + (Iterable) dictionary.getValue(entity, relationship.getName(), scope)).allInMemory().build(); } @Override - public Iterable loadObjects(EntityProjection projection, - RequestScope scope) { + public DataStoreIterable loadObjects(EntityProjection projection, + RequestScope scope) { synchronized (dataStore) { Map data = dataStore.get(projection.getType()); - return data.values(); + return new DataStoreIterableBuilder<>(data.values()).allInMemory().build(); } } @@ -163,21 +165,6 @@ public void close() throws IOException { operations.clear(); } - @Override - public FeatureSupport supportsFiltering(RequestScope scope, Optional parent, EntityProjection projection) { - return FeatureSupport.NONE; - } - - @Override - public boolean supportsSorting(RequestScope scope, Optional parent, EntityProjection projection) { - return false; - } - - @Override - public boolean supportsPagination(RequestScope scope, Optional parent, EntityProjection projection) { - return false; - } - private boolean containsObject(Object obj) { return containsObject(EntityDictionary.getType(obj), obj); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java index d3358737f7..4cf1da1aff 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/inmemory/InMemoryStoreTransaction.java @@ -9,11 +9,12 @@ import com.yahoo.elide.core.Path; import com.yahoo.elide.core.PersistentResource; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; +import com.yahoo.elide.core.datastore.DataStoreIterableBuilder; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.FilterPredicatePushdownExtractor; import com.yahoo.elide.core.filter.expression.InMemoryExecutionVerifier; -import com.yahoo.elide.core.filter.expression.InMemoryFilterExecutor; import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; import com.yahoo.elide.core.request.Pagination; @@ -27,11 +28,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -64,24 +65,23 @@ public class InMemoryStoreTransaction implements DataStoreTransaction { */ @FunctionalInterface private interface DataFetcher { - Object fetch(Optional filterExpression, - Optional sorting, - Optional pagination, - RequestScope scope); + DataStoreIterable fetch(Optional filterExpression, + Optional sorting, + Optional pagination, + RequestScope scope); } - public InMemoryStoreTransaction(DataStoreTransaction tx) { this.tx = tx; } @Override - public Object getRelation(DataStoreTransaction relationTx, - Object entity, - Relationship relationship, - RequestScope scope) { + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, + Object entity, + Relationship relationship, + RequestScope scope) { DataFetcher fetcher = (filterExpression, sorting, pagination, requestScope) -> - tx.getRelation(relationTx, entity, relationship.copyOf() + tx.getToManyRelation(relationTx, entity, relationship.copyOf() .projection(relationship.getProjection().copyOf() .filterExpression(filterExpression.orElse(null)) .sorting(sorting.orElse(null)) @@ -95,23 +95,22 @@ public Object getRelation(DataStoreTransaction relationTx, * It must be done in memory by Elide as some newly created entities have not yet been persisted. */ boolean filterInMemory = scope.getNewPersistentResources().size() > 0; - return fetchData(fetcher, Optional.of(entity), relationship.getProjection(), filterInMemory, scope); + return fetchData(fetcher, relationship.getProjection(), filterInMemory, scope); } @Override public Object loadObject(EntityProjection projection, Serializable id, RequestScope scope) { - - if (projection.getFilterExpression() == null - || tx.supportsFiltering(scope, Optional.empty(), projection) == FeatureSupport.FULL) { + if (projection.getFilterExpression() == null) { return tx.loadObject(projection, id, scope); } + return DataStoreTransaction.super.loadObject(projection, id, scope); } @Override - public Iterable loadObjects(EntityProjection projection, + public DataStoreIterable loadObjects(EntityProjection projection, RequestScope scope) { DataFetcher fetcher = (filterExpression, sorting, pagination, requestScope) -> @@ -121,7 +120,7 @@ public Iterable loadObjects(EntityProjection projection, .sorting(sorting.orElse(null)) .build(), requestScope); - return (Iterable) fetchData(fetcher, Optional.empty(), projection, false, scope); + return fetchData(fetcher, projection, false, scope); } @Override @@ -144,6 +143,15 @@ public T createNewObject(Type entityClass, RequestScope scope) { return tx.createNewObject(entityClass, scope); } + @Override + public R getToOneRelation( + DataStoreTransaction relationTx, + T entity, Relationship relationship, + RequestScope scope + ) { + return tx.getToOneRelation(relationTx, entity, relationship, scope); + } + @Override public void close() throws IOException { tx.close(); @@ -193,85 +201,104 @@ public void createObject(Object entity, RequestScope scope) { tx.createObject(entity, scope); } - private Iterable filterLoadedData(Iterable loadedRecords, + private DataStoreIterable filterLoadedData(DataStoreIterable loadedRecords, Optional filterExpression, RequestScope scope) { + if (! filterExpression.isPresent()) { return loadedRecords; } - Predicate predicate = filterExpression.get().accept(new InMemoryFilterExecutor(scope)); + return new DataStoreIterable<>() { + @Override + public Iterable getWrappedIterable() { + return loadedRecords; + } + + @Override + public Iterator iterator() { + return new FilteredIterator<>(filterExpression.get(), scope, loadedRecords.iterator()); + } - return StreamSupport.stream(loadedRecords.spliterator(), false) - .filter(predicate::test) - .collect(Collectors.toList()); - } + @Override + public boolean needsInMemoryFilter() { + return true; + } + + @Override + public boolean needsInMemorySort() { + return true; + } - private Object fetchData(DataFetcher fetcher, - Optional parent, - EntityProjection projection, - boolean filterInMemory, - RequestScope scope) { + @Override + public boolean needsInMemoryPagination() { + return true; + } + }; + } + private DataStoreIterable fetchData( + DataFetcher fetcher, + EntityProjection projection, + boolean filterInMemory, + RequestScope scope + ) { Optional filterExpression = Optional.ofNullable(projection.getFilterExpression()); Pair, Optional> expressionSplit = splitFilterExpression( - scope, parent, projection, filterInMemory); + scope, projection, filterInMemory); Optional dataStoreFilter = expressionSplit.getLeft(); Optional inMemoryFilter = expressionSplit.getRight(); - Pair, Optional> sortSplit = splitSorting(scope, parent, - projection, inMemoryFilter.isPresent()); - - Optional dataStoreSort = sortSplit.getLeft(); - Optional inMemorySort = sortSplit.getRight(); + Optional dataStoreSorting = getDataStoreSorting(scope, projection, filterInMemory); - Pair, Optional> paginationSplit = splitPagination(scope, parent, - projection, inMemoryFilter.isPresent(), inMemorySort.isPresent()); + boolean sortingInMemory = dataStoreSorting.isEmpty() && projection.getSorting() != null; - Optional dataStorePagination = paginationSplit.getLeft(); - Optional inMemoryPagination = paginationSplit.getRight(); + Optional dataStorePagination = inMemoryFilter.isPresent() || sortingInMemory + ? Optional.empty() : Optional.ofNullable(projection.getPagination()); - Object result = fetcher.fetch(dataStoreFilter, dataStoreSort, dataStorePagination, scope); + DataStoreIterable loadedRecords = + fetcher.fetch(dataStoreFilter, dataStoreSorting, dataStorePagination, scope); - if (! (result instanceof Iterable)) { - return result; + if (loadedRecords == null) { + return new DataStoreIterableBuilder().build(); } - Iterable loadedRecords = (Iterable) result; - - if (inMemoryFilter.isPresent()) { + if (inMemoryFilter.isPresent() || (loadedRecords.needsInMemoryFilter() + && projection.getFilterExpression() != null)) { loadedRecords = filterLoadedData(loadedRecords, filterExpression, scope); } - return sortAndPaginateLoadedData( loadedRecords, - inMemorySort, - inMemoryPagination, + sortingInMemory, + projection.getSorting(), + projection.getPagination(), scope); } + private DataStoreIterable sortAndPaginateLoadedData( + DataStoreIterable loadedRecords, + boolean sortingInMemory, + Sorting sorting, + Pagination pagination, + RequestScope scope + ) { + + Map sortRules = sorting == null ? new HashMap<>() : sorting.getSortingPaths(); + + boolean mustSortInMemory = ! sortRules.isEmpty() + && (sortingInMemory || loadedRecords.needsInMemorySort()); - private Iterable sortAndPaginateLoadedData(Iterable loadedRecords, - Optional sorting, - Optional pagination, - RequestScope scope) { + boolean mustPaginateInMemory = pagination != null + && (mustSortInMemory || loadedRecords.needsInMemoryPagination()); //Try to skip the data copy if possible - if (! sorting.isPresent() && ! pagination.isPresent()) { + if (! mustSortInMemory && ! mustPaginateInMemory) { return loadedRecords; } - Map sortRules = sorting - .map(Sorting::getSortingPaths) - .orElseGet(HashMap::new); - - // No sorting required for this type & no pagination. - if (sortRules.isEmpty() && ! pagination.isPresent()) { - return loadedRecords; - } //We need an in memory copy to sort or paginate. List results = StreamSupport.stream(loadedRecords.spliterator(), false).collect(Collectors.toList()); @@ -279,11 +306,11 @@ private Iterable sortAndPaginateLoadedData(Iterable loadedRecord results = sortInMemory(results, sortRules, scope); } - if (pagination.isPresent()) { - results = paginateInMemory(results, pagination.get()); + if (pagination != null) { + results = paginateInMemory(results, pagination); } - return results; + return new DataStoreIterableBuilder(results).build(); } private List paginateInMemory(List records, Pagination pagination) { @@ -345,18 +372,51 @@ private Comparator getComparator(Path path, Sorting.SortOrder order, Req } /** - * Splits a filter expression into two components: + * Returns the sorting (if any) that should be pushed to the datastore. + * @param scope The request context + * @param projection The projection being loaded. + * @param filterInMemory Whether or not the transaction requires in memory filtering. + * @return An optional sorting. + */ + private Optional getDataStoreSorting( + RequestScope scope, + EntityProjection projection, + boolean filterInMemory + ) { + Sorting sorting = projection.getSorting(); + if (filterInMemory) { + return Optional.empty(); + } + Map sortRules = sorting == null ? new HashMap<>() : sorting.getSortingPaths(); + + boolean sortingOnComputedAttribute = false; + for (Path path: sortRules.keySet()) { + if (path.isComputed(scope.getDictionary())) { + Type pathType = path.getPathElements().get(0).getType(); + if (projection.getType().equals(pathType)) { + sortingOnComputedAttribute = true; + break; + } + } + } + if (sortingOnComputedAttribute) { + return Optional.empty(); + } else { + return Optional.ofNullable(sorting); + } + } + + /** + * Splits a filter expression into two components. They are: * - a component that should be pushed down to the data store * - a component that should be executed in memory * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. * @param projection The projection being loaded. * @param filterInMemory Whether or not the transaction requires in memory filtering. * @return A pair of filter expressions (data store expression, in memory expression) */ private Pair, Optional> splitFilterExpression( RequestScope scope, - Optional parent, EntityProjection projection, boolean filterInMemory ) { @@ -368,11 +428,7 @@ private Pair, Optional> splitFilter boolean transactionNeedsInMemoryFiltering = filterInMemory; if (filterExpression.isPresent()) { - FeatureSupport filterSupport = tx.supportsFiltering(scope, parent, projection); - - boolean storeNeedsInMemoryFiltering = filterSupport != FeatureSupport.FULL; - - if (transactionNeedsInMemoryFiltering || filterSupport == FeatureSupport.NONE) { + if (transactionNeedsInMemoryFiltering) { inStoreFilterExpression = Optional.empty(); } else { inStoreFilterExpression = Optional.ofNullable( @@ -383,7 +439,7 @@ private Pair, Optional> splitFilter boolean expressionNeedsInMemoryFiltering = InMemoryExecutionVerifier.shouldExecuteInMemory( scope.getDictionary(), filterExpression.get()); - if (transactionNeedsInMemoryFiltering || storeNeedsInMemoryFiltering || expressionNeedsInMemoryFiltering) { + if (transactionNeedsInMemoryFiltering || expressionNeedsInMemoryFiltering) { inMemoryFilterExpression = filterExpression; } } @@ -391,68 +447,6 @@ private Pair, Optional> splitFilter return Pair.of(inStoreFilterExpression, inMemoryFilterExpression); } - /** - * Splits a sorting object into two components: - * - a component that should be pushed down to the data store - * - a component that should be executed in memory - * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. - * @param projection The projection being loaded. - * @param filteredInMemory Whether or not filtering was performed in memory - * @return A pair of sorting objects (data store sort, in memory sort) - */ - private Pair, Optional> splitSorting( - RequestScope scope, - Optional parent, - EntityProjection projection, - boolean filteredInMemory - ) { - Optional sorting = Optional.ofNullable(projection.getSorting()); - - if (sorting.isPresent() && (! tx.supportsSorting(scope, parent, projection) || filteredInMemory)) { - return Pair.of(Optional.empty(), sorting); - } - return Pair.of(sorting, Optional.empty()); - } - - /** - * Splits a pagination object into two components: - * - a component that should be pushed down to the data store - * - a component that should be executed in memory - * @param scope The request context - * @param parent If this is a relationship load, the parent object. Otherwise not set. - * @param projection The projection being loaded. - * @param filteredInMemory Whether or not filtering was performed in memory - * @param sortedInMemory Whether or not sorting was performed in memory - * @return A pair of pagination objects (data store pagination, in memory pagination) - */ - private Pair, Optional> splitPagination( - RequestScope scope, - Optional parent, - EntityProjection projection, - boolean filteredInMemory, - boolean sortedInMemory - ) { - - Optional pagination = Optional.ofNullable(projection.getPagination()); - - if (!tx.supportsPagination(scope, parent, projection) - || filteredInMemory - || sortedInMemory) { - return Pair.of(Optional.empty(), pagination); - - /* - * For default pagination, we let the store do its work, but we also let the store ignore pagination - * by also performing in memory. This allows the ORM the opportunity to manage its own SQL query generation - * to avoid N+1. - */ - } else if (pagination.isPresent() && pagination.get().isDefaultInstance()) { - return Pair.of(pagination, pagination); - } else { - return Pair.of(pagination, Optional.empty()); - } - } - @Override public void cancel(RequestScope scope) { tx.cancel(scope); diff --git a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java index 220757e617..dd9781ef8a 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/datastore/wrapped/TransactionWrapper.java @@ -7,6 +7,7 @@ package com.yahoo.elide.core.datastore.wrapped; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.core.datastore.DataStoreIterable; import com.yahoo.elide.core.datastore.DataStoreTransaction; import com.yahoo.elide.core.request.Attribute; import com.yahoo.elide.core.request.EntityProjection; @@ -16,7 +17,6 @@ import java.io.IOException; import java.io.Serializable; -import java.util.Optional; import java.util.Set; /** @@ -39,9 +39,15 @@ public T loadObject(EntityProjection projection, Serializable id, } @Override - public R getRelation(DataStoreTransaction relationTx, T entity, - Relationship relationship, RequestScope scope) { - return tx.getRelation(relationTx, entity, relationship, scope); + public DataStoreIterable getToManyRelation(DataStoreTransaction relationTx, T entity, + Relationship relationship, RequestScope scope) { + return tx.getToManyRelation(relationTx, entity, relationship, scope); + } + + @Override + public R getToOneRelation(DataStoreTransaction relationTx, T entity, + Relationship relationship, RequestScope scope) { + return tx.getToOneRelation(relationTx, entity, relationship, scope); } @Override @@ -68,21 +74,6 @@ public void setAttribute(T entity, Attribute attribute, RequestScope scope) tx.setAttribute(entity, attribute, scope); } - @Override - public FeatureSupport supportsFiltering(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsFiltering(scope, parent, projection); - } - - @Override - public boolean supportsSorting(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsSorting(scope, parent, projection); - } - - @Override - public boolean supportsPagination(RequestScope scope, Optional parent, EntityProjection projection) { - return tx.supportsPagination(scope, parent, projection); - } - @Override public void save(T o, RequestScope requestScope) { tx.save(o, requestScope); @@ -109,7 +100,7 @@ public void createObject(Object o, RequestScope requestScope) { } @Override - public Iterable loadObjects(EntityProjection projection, RequestScope scope) { + public DataStoreIterable loadObjects(EntityProjection projection, RequestScope scope) { return tx.loadObjects(projection, scope); } diff --git a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java index 29ae71c304..7024a6a66d 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/ArgumentType.java @@ -6,6 +6,7 @@ package com.yahoo.elide.core.dictionary; import com.yahoo.elide.core.type.Type; +import lombok.Builder; import lombok.Value; /** @@ -21,6 +22,7 @@ public ArgumentType(String name, Type type) { this(name, type, null); } + @Builder public ArgumentType(String name, Type type, Object defaultValue) { this.name = name; this.type = type; diff --git a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java index d3524d38b4..863e4fbe97 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/dictionary/EntityBinding.java @@ -67,6 +67,8 @@ * @see com.yahoo.elide.annotation.Include#name */ public class EntityBinding { + public static final List