From 7cbfb7afe121efe91e28f9a210df566463379ddf Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 2 Nov 2026 11:11:47 +0000 Subject: [PATCH 01/16] Support serializing and executing explodes as table functions over an ExplodeValue Adds what a server needs to deserialize and execute plans in which explodes are table functions over an `ExplodeValue`, ahead of the planner producing them: - `ExplodeValue`, a streaming value producing a struct `(element)`, or `(element, ordinal)` if it is with ordinality, per element of an array, and its continuation. - `ArraySortValue`, a sorted array, which a sorted IN source explodes. - `InExplodeSource`, an IN source binding the structs of an explode. - Serialization of list literals in `LiteralValue`. - `RecordQueryInUnionPlan` reports the dynamic types of its sources, and `InComparandSource` tolerates comparisons without a comparand value. Nothing plans these yet. --- .../plan/cascades/values/ArraySortValue.java | 207 +++++++ .../plan/cascades/values/ExplodeValue.java | 534 ++++++++++++++++++ .../plan/cascades/values/LiteralValue.java | 50 +- .../query/plan/plans/InComparandSource.java | 5 +- .../query/plan/plans/InExplodeSource.java | 232 ++++++++ .../plan/plans/RecordQueryInUnionPlan.java | 10 + .../src/main/proto/record_cursor.proto | 4 + .../src/main/proto/record_query_plan.proto | 21 + .../cascades/values/ExplodeValueTest.java | 392 +++++++++++++ .../query/plan/plans/InExplodeSourceTest.java | 127 +++++ 10 files changed, 1574 insertions(+), 8 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java new file mode 100644 index 00000000000..0f6478348cb --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java @@ -0,0 +1,207 @@ +/* + * ArraySortValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PArraySortValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A value that returns a copy of its child array {@link Value} with its elements sorted in ascending or, if reverse, + * descending order. The elements must be comparable with each other. + */ +@API(API.Status.EXPERIMENTAL) +public class ArraySortValue extends AbstractValue implements ValueWithChild { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Array-Sort-Value"); + + @SuppressWarnings("unchecked") + private static final Comparator ELEMENT_COMPARATOR = Comparator.comparing(Comparable.class::cast); + + @Nonnull + private final Value childValue; + private final boolean reverse; + + public ArraySortValue(@Nonnull final Value childValue, final boolean reverse) { + Verify.verify(Objects.requireNonNull(childValue.getResultType()).isArray()); + this.childValue = childValue; + this.reverse = reverse; + } + + public boolean isReverse() { + return reverse; + } + + @Nonnull + @Override + public List computeChildren() { + return ImmutableList.of(childValue); + } + + @Nonnull + @Override + public Value getChild() { + return childValue; + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public ValueWithChild withNewChild(@Nonnull final Value rebasedChild) { + if (getChild() == rebasedChild) { + return this; + } + return new ArraySortValue(rebasedChild, reverse); + } + + @Nonnull + @Override + public Type getResultType() { + return childValue.getResultType(); + } + + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final var childResult = childValue.eval(store, context); + if (childResult == null) { + return null; + } + return sort((List)childResult, reverse); + } + + /** + * Sorts a list of comparable elements. + * @param elements the elements to sort + * @param reverse whether to sort in descending order + * @return a sorted copy of the elements + */ + @Nonnull + public static List sort(@Nonnull final List elements, final boolean reverse) { + final List sorted = new ArrayList<>(elements); + sorted.sort(reverse ? ELEMENT_COMPARATOR.reversed() : ELEMENT_COMPARATOR); + return sorted; + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH, reverse); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, childValue, reverse); + } + + @Nonnull + @Override + public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { + return super.equalsWithoutChildren(other) + .filter(ignored -> reverse == ((ArraySortValue)other).isReverse()); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + final var explainTokens = new ExplainTokens().addFunctionCall("arraySort", + Value.explainFunctionArguments(explainSuppliers)); + if (reverse) { + explainTokens.addWhitespace().addKeyword("DESC"); + } + return ExplainTokensWithPrecedence.of(explainTokens); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Nonnull + @Override + public PArraySortValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PArraySortValue.newBuilder() + .setChildValue(childValue.toValueProto(serializationContext)) + .setReverse(reverse) + .build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull PlanSerializationContext serializationContext) { + return PValue.newBuilder().setArraySortValue(toProto(serializationContext)).build(); + } + + @Nonnull + public static ArraySortValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PArraySortValue arraySortValueProto) { + return new ArraySortValue( + Value.fromValueProto(serializationContext, Objects.requireNonNull(arraySortValueProto.getChildValue())), + arraySortValueProto.getReverse()); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PArraySortValue.class; + } + + @Nonnull + @Override + public ArraySortValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PArraySortValue arraySortValueProto) { + return ArraySortValue.fromProto(serializationContext, arraySortValueProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java new file mode 100644 index 00000000000..7f11d0949d6 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -0,0 +1,534 @@ +/* + * ExplodeValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.Bindings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorProto; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorVisitor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PExplodeValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.properties.CardinalitiesProperty; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.record.util.ProtoUtils; +import com.apple.foundationdb.tuple.ByteArrayUtil2; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.ProtocolMessageEnum; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.IntStream; + +/** + * A {@link StreamingValue} that iterates over the elements of an array. For every element it produces a struct whose + * first anonymous field is the element. If the explode is {@linkplain #hasOrdinality() with ordinality}, the struct has + * a second anonymous field that holds the 0-based ordinal of the element within the array, akin to SQL's + * {@code UNNEST ... WITH ORDINALITY}. + *

+ * An explode without ordinality does not expose the positions of the elements, so its collection can be replaced by + * any permutation of it, for example by a sorted one. + *

+ */ +@API(API.Status.EXPERIMENTAL) +public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Explode-Value"); + + /** + * The name of the field of the struct an explode produces that holds the element. + */ + @Nonnull + public static final String ELEMENT_FIELD_NAME = "_0"; + + /** + * The type of the ordinal. + */ + @Nonnull + private static final Type ORDINAL_TYPE = Type.primitiveType(Type.TypeCode.INT, false); + + /** + * The collection value. Must evaluate to an array type. + */ + @Nonnull + private final Value collectionValue; + + /** + * The element type of the collection. + */ + @Nonnull + private final Type elementType; + + /** + * Whether the struct this explode produces carries the ordinal of the element. + */ + private final boolean withOrdinality; + + /** + * The alias the element being produced is bound to while its struct is evaluated. + */ + @Nonnull + private final CorrelationIdentifier elementAlias; + + /** + * The alias the ordinal of the element being produced is bound to while its struct is evaluated. Unused if this + * explode is not with ordinality. + */ + @Nonnull + private final CorrelationIdentifier ordinalAlias; + + /** + * The struct produced per element: a record constructor of the element and, if with ordinality, its ordinal. + */ + @Nonnull + private final RecordConstructorValue structValue; + + public ExplodeValue(@Nonnull final Value collectionValue) { + this(collectionValue, false); + } + + public ExplodeValue(@Nonnull final Value collectionValue, final boolean withOrdinality) { + Verify.verify(collectionValue.getResultType().isArray()); + this.collectionValue = collectionValue; + this.withOrdinality = withOrdinality; + this.elementType = Objects.requireNonNull(((Type.Array)collectionValue.getResultType()).getElementType()); + this.elementAlias = CorrelationIdentifier.uniqueId(); + this.ordinalAlias = CorrelationIdentifier.uniqueId(); + final ImmutableList.Builder> columnsBuilder = ImmutableList.builder(); + columnsBuilder.add(Column.unnamedOf(QuantifiedObjectValue.of(elementAlias, elementType))); + if (withOrdinality) { + columnsBuilder.add(Column.unnamedOf(QuantifiedObjectValue.of(ordinalAlias, ORDINAL_TYPE))); + } + this.structValue = RecordConstructorValue.ofColumns(columnsBuilder.build(), false); + Verify.verify(ELEMENT_FIELD_NAME.equals(structValue.getResultType().getField(0).getFieldName())); + } + + @Nonnull + public Value getCollectionValue() { + return collectionValue; + } + + public boolean hasOrdinality() { + return withOrdinality; + } + + /** + * Returns an explode of a different collection that is with ordinality if and only if this one is. + * @param newCollectionValue the new collection + * @return an explode of the new collection + */ + @Nonnull + public ExplodeValue withCollectionValue(@Nonnull final Value newCollectionValue) { + return new ExplodeValue(newCollectionValue, withOrdinality); + } + + @Nonnull + public Type getElementType() { + return elementType; + } + + /** + * Returns the element of a struct produced by an explode. + * + * @param explodedValue a value of the {@linkplain #getResultType() result type} of an explode, typically the + * flowed object value of a quantifier ranging over the explode + * @return the element the struct carries + */ + @Nonnull + public static Value elementValue(@Nonnull final Value explodedValue) { + return FieldValue.ofOrdinalNumberAndFuseIfPossible(explodedValue, 0); + } + + /** + * Returns the 0-based ordinal of a struct produced by an explode {@linkplain #hasOrdinality() with ordinality}. + * + * @param explodedValue a value of the {@linkplain #getResultType() result type} of an explode with ordinality, + * typically the flowed object value of a quantifier ranging over the explode + * @return the ordinal the struct carries + */ + @Nonnull + public static Value ordinalValue(@Nonnull final Value explodedValue) { + return FieldValue.ofOrdinalNumberAndFuseIfPossible(explodedValue, 1); + } + + @Nonnull + @Override + public Type.Record getResultType() { + return structValue.getResultType(); + } + + @Nonnull + @Override + public RecordCursor evalAsStream(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + return new Cursor(store.getExecutor(), elementsOf(store, context), + (elements, ordinal) -> structAt(context, elements, ordinal), continuation) + .skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); + } + + /** + * Evaluates this explode eagerly, producing the structs {@link #evalAsStream} would stream. + * @param store the record store, may be {@code null} if the collection does not need one + * @param context the evaluation context + * @return the structs of the elements of the collection, in the order of the collection + */ + @Nonnull + public List evalAsList(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final List elements = elementsOf(store, context); + return IntStream.range(0, elements.size()) + .mapToObj(ordinal -> structAt(context, elements, ordinal)) + .collect(ImmutableList.toImmutableList()); + } + + /** + * Evaluates the collection. A {@code null} collection has no elements. + */ + @Nonnull + private List elementsOf(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final Object result = collectionValue.eval(store, context); + return result == null ? List.of() : (List)result; + } + + @Nonnull + private Object structAt(@Nonnull final EvaluationContext context, @Nonnull final List elements, final int ordinal) { + return structOf(context, Verify.verifyNotNull(elements.get(ordinal), "array elements must be non-null"), ordinal); + } + + @Nonnull + private Object structOf(@Nonnull final EvaluationContext context, @Nonnull final Object element, final int ordinal) { + EvaluationContext elementContext = context + .withBinding(Bindings.Internal.CORRELATION, elementAlias, + QueryResult.ofComputed(coerceElement(context.getTypeRepository(), element))); + if (withOrdinality) { + elementContext = elementContext.withBinding(Bindings.Internal.CORRELATION, ordinalAlias, + QueryResult.ofComputed(ordinal)); + } + return Objects.requireNonNull(structValue.eval(null, elementContext)); + } + + /** + * Coerces an element to the representation of the element type in the struct. Elements of literal lists, such + * as the lists of IN predicates, are plain Java objects that do not necessarily use that representation: numbers + * may be narrower than the element type, and enums are Java enums or {@link ProtoUtils.DynamicEnum}s rather than + * enum value descriptors. + */ + @Nonnull + private Object coerceElement(@Nonnull final TypeRepository typeRepository, @Nonnull final Object element) { + final Type elementType = structValue.getResultType().getField(0).getFieldType(); + if (elementType instanceof Type.Enum) { + if (element instanceof ProtocolMessageEnum) { + return ((ProtocolMessageEnum)element).getValueDescriptor(); + } + if (element instanceof ProtoUtils.DynamicEnum) { + return Objects.requireNonNull(typeRepository.getEnumValue(typeRepository.getProtoTypeName(elementType), + ((ProtoUtils.DynamicEnum)element).getName())); + } + return element; + } + if (element instanceof final Number number && elementType.isPrimitive()) { + return switch (elementType.getTypeCode()) { + case LONG -> element instanceof Long ? element : (Object)number.longValue(); + case DOUBLE -> element instanceof Double ? element : (Object)number.doubleValue(); + case FLOAT -> element instanceof Float ? element : (Object)number.floatValue(); + default -> element; + }; + } + return element; + } + + @Nonnull + @Override + public CardinalitiesProperty.Cardinalities getCardinalities() { + final int size; + if (collectionValue instanceof AbstractArrayConstructorValue) { + size = Iterables.size(collectionValue.getChildren()); + } else if (collectionValue instanceof LiteralValue && + ((LiteralValue)collectionValue).getLiteralValue() instanceof List) { + size = ((List)((LiteralValue)collectionValue).getLiteralValue()).size(); + } else { + return CardinalitiesProperty.Cardinalities.unknownMaxCardinality(); + } + final var cardinality = CardinalitiesProperty.Cardinality.ofCardinality(size); + return new CardinalitiesProperty.Cardinalities(cardinality, cardinality); + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + throw new IllegalStateException("unable to eval an streaming value with eval()"); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + return ImmutableList.of(collectionValue); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public Value withChildren(final Iterable newChildren) { + Verify.verify(Iterables.size(newChildren) == 1); + final Value newCollectionValue = Iterables.getOnlyElement(newChildren); + if (newCollectionValue == collectionValue) { + return this; + } + return withCollectionValue(newCollectionValue); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("explode", + Iterables.getOnlyElement(explainSuppliers).get().getExplainTokens())); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH, withOrdinality); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, collectionValue, withOrdinality); + } + + @Nonnull + @Override + public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { + return super.equalsWithoutChildren(other) + .filter(ignored -> withOrdinality == ((ExplodeValue)other).hasOrdinality()); + } + + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @Nonnull + @Override + public PExplodeValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PExplodeValue.newBuilder() + .setCollectionValue(collectionValue.toValueProto(serializationContext)) + .setWithOrdinality(withOrdinality) + .build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + return PValue.newBuilder().setExplodeValue(toProto(serializationContext)).build(); + } + + @Nonnull + public static ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PExplodeValue explodeValueProto) { + return new ExplodeValue(Value.fromValueProto(serializationContext, + Objects.requireNonNull(explodeValueProto.getCollectionValue())), explodeValueProto.getWithOrdinality()); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PExplodeValue.class; + } + + @Nonnull + @Override + public ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PExplodeValue explodeValueProto) { + return ExplodeValue.fromProto(serializationContext, explodeValueProto); + } + } + + /** + * A cursor over the elements of an array, producing a struct of each element and its ordinal. A struct is only + * created once the cursor reaches its element. The continuation records the position of the next element. + */ + public static class Cursor implements RecordCursor { + @Nonnull + private final Executor executor; + + @Nonnull + private final List elements; + + /** + * Creates the struct of the element at an ordinal of the array, and that ordinal. + */ + @Nonnull + private final BiFunction, Integer, Object> structCreator; + + private int nextPosition; // position of the next element to return + + private boolean closed = false; + + Cursor(@Nonnull final Executor executor, @Nonnull final List elements, + @Nonnull final BiFunction, Integer, Object> structCreator, @Nullable final byte[] continuation) { + this(executor, elements, structCreator, + continuation == null ? 0 : Continuation.from(continuation).getNextPosition()); + } + + private Cursor(@Nonnull final Executor executor, @Nonnull final List elements, + @Nonnull final BiFunction, Integer, Object> structCreator, final int nextPosition) { + if (nextPosition < 0 || nextPosition > elements.size()) { + throw new RecordCoreException("invalid continuation position for explode") + .addLogInfo("nextPosition", nextPosition) + .addLogInfo("arraySize", elements.size()); + } + this.executor = executor; + this.elements = elements; + this.structCreator = structCreator; + this.nextPosition = nextPosition; + } + + @Nonnull + @Override + public CompletableFuture> onNext() { + return CompletableFuture.completedFuture(getNext()); + } + + @Nonnull + @Override + public RecordCursorResult getNext() { + if (nextPosition >= elements.size()) { + return RecordCursorResult.exhausted(); + } + final int ordinal = nextPosition; + nextPosition++; + return RecordCursorResult.withNextValue( + QueryResult.ofComputed(structCreator.apply(elements, ordinal)), + new Continuation(nextPosition)); + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public boolean accept(@Nonnull final RecordCursorVisitor visitor) { + visitor.visitEnter(this); + return visitor.visitLeave(this); + } + + @Override + @Nonnull + public Executor getExecutor() { + return executor; + } + + private static class Continuation implements RecordCursorContinuation { + private final int nextPosition; + + Continuation(final int nextPosition) { + this.nextPosition = nextPosition; + } + + @Override + public boolean isEnd() { + // A continuation returned alongside a value must not be an end continuation, per the contract of + // RecordCursorResult, even if the cursor knows there is nothing after that value. Exhaustion is + // signalled by the exhausted result, which carries the end continuation. + return false; + } + + public int getNextPosition() { + return nextPosition; + } + + @Nonnull + @Override + public ByteString toByteString() { + return RecordCursorProto.ExplodeCursorContinuation.newBuilder() + .setNextPosition(nextPosition) + .build() + .toByteString(); + } + + @Nullable + @Override + public byte[] toBytes() { + return toByteString().toByteArray(); + } + + @Nonnull + public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes) { + try { + final var parsed = RecordCursorProto.ExplodeCursorContinuation.parseFrom(unparsedContinuationBytes); + return new Continuation(parsed.getNextPosition()); + } catch (InvalidProtocolBufferException ex) { + throw new RecordCoreException("invalid continuation", ex) + .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(unparsedContinuationBytes)); + } + } + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java index dda428e3524..830fec1c746 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializable; import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PComparableObject; import com.apple.foundationdb.record.planprotos.PLiteralValue; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; @@ -46,6 +47,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -118,13 +120,31 @@ public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { return LeafValue.super.equalsWithoutChildren(other) .filter(ignored -> { final LiteralValue that = (LiteralValue)other; - if (value == null && that.value == null) { - return true; - } - return Boolean.TRUE.equals(Comparisons.evalComparison(Comparisons.Type.EQUALS, value, that.value)); + return literalsEqual(value, that.value); }); } + private static boolean literalsEqual(@Nullable final Object value, @Nullable final Object otherValue) { + if (value == null && otherValue == null) { + return true; + } + if (value instanceof List && otherValue instanceof List) { + // compare element-wise, so that elements that do not round-trip as themselves, e.g. enums, still compare + final List elements = (List)value; + final List otherElements = (List)otherValue; + if (elements.size() != otherElements.size()) { + return false; + } + for (int i = 0; i < elements.size(); i++) { + if (!literalsEqual(elements.get(i), otherElements.get(i))) { + return false; + } + } + return true; + } + return Boolean.TRUE.equals(Comparisons.evalComparison(Comparisons.Type.EQUALS, value, otherValue)); + } + @Override public int hashCodeWithoutChildren() { return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, value); @@ -147,7 +167,14 @@ public PLiteralValue toProto(@Nonnull final PlanSerializationContext serializati final var builder = PLiteralValue.newBuilder(); // TODO there should be no punishment if the type is serialized alongside with the value final var resultTypeProto = resultType.isVector() ? resultType.toTypeProto(serializationContext) : null; - builder.setValue(PlanSerialization.valueObjectToProto(value, resultTypeProto)); + if (value instanceof List) { + builder.setIsList(true); + for (final Object element : (List)value) { + builder.addListElements(PlanSerialization.valueObjectToProto(element)); + } + } else { + builder.setValue(PlanSerialization.valueObjectToProto(value, resultTypeProto)); + } builder.setResultType(resultType.toTypeProto(serializationContext)); return builder.build(); } @@ -165,8 +192,17 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization @Nonnull public static LiteralValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PLiteralValue literalValueProto) { - return new LiteralValue<>(Type.fromTypeProto(serializationContext, literalValueProto.getResultType()), - (T)PlanSerialization.protoToValueObject(literalValueProto.getValue())); + final Object value; + if (literalValueProto.getIsList()) { + final List elements = new ArrayList<>(literalValueProto.getListElementsCount()); + for (final PComparableObject elementProto : literalValueProto.getListElementsList()) { + elements.add(PlanSerialization.protoToValueObject(elementProto)); + } + value = elements; + } else { + value = PlanSerialization.protoToValueObject(literalValueProto.getValue()); + } + return new LiteralValue<>(Type.fromTypeProto(serializationContext, literalValueProto.getResultType()), (T)value); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java index 5b14744151b..391d80dc038 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java @@ -32,10 +32,12 @@ import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.Message; import javax.annotation.Nonnull; @@ -180,7 +182,8 @@ public Type getResultType() { @Nonnull @Override public Set getDynamicTypes() { - return Objects.requireNonNull(comparison.getValue()).getDynamicTypes(); + final Value comparandValue = comparison.getValue(); + return comparandValue == null ? ImmutableSet.of() : comparandValue.getDynamicTypes(); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java new file mode 100644 index 00000000000..4a168991b4b --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -0,0 +1,232 @@ +/* + * InExplodeSource.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PInExplodeSource; +import com.apple.foundationdb.record.planprotos.PInSource; +import com.apple.foundationdb.record.query.expressions.Comparisons; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; +import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Source of values for an "IN" query that is planned from a quantifier ranging over an {@link ExplodeValue}. Like + * the explode itself, it binds the struct of each element of the collection, so the correlated inner plan can read + * the element and, if any, the ordinal the same way it would read them from the explode. + *

+ * The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection would + * renumber the ordinals, which is why only the collections of explodes that are not + * {@linkplain ExplodeValue#hasOrdinality() with ordinality} may be sorted. + *

+ */ +@API(API.Status.INTERNAL) +public class InExplodeSource extends InComparandSource { + @Nonnull + private static final ObjectPlanHash OBJECT_PLAN_HASH_IN_EXPLODE_SOURCE = new ObjectPlanHash("In-Explode"); + + @Nonnull + private final ExplodeValue explodeValue; + + protected InExplodeSource(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PInExplodeSource inExplodeSourceProto) { + super(serializationContext, Objects.requireNonNull(inExplodeSourceProto.getSuper())); + this.explodeValue = (ExplodeValue)Value.fromValueProto(serializationContext, + Objects.requireNonNull(inExplodeSourceProto.getExplodeValue())); + } + + public InExplodeSource(@Nonnull final String bindingName, @Nonnull final ExplodeValue explodeValue) { + super(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue.getCollectionValue())); + Verify.verify(!explodeValue.hasOrdinality() || !(explodeValue.getCollectionValue() instanceof ArraySortValue)); + this.explodeValue = explodeValue; + } + + /** + * Returns the collection the explode of this source ranges over before it is sorted. + * @return the unsorted collection + */ + @Nonnull + private Value getUnsortedCollectionValue() { + final Value collectionValue = explodeValue.getCollectionValue(); + return collectionValue instanceof ArraySortValue ? ((ArraySortValue)collectionValue).getChild() : collectionValue; + } + + /** + * Returns the name of the parameter this source explodes, if it explodes a parameter. + * @return the parameter name or {@code null} + */ + @Nullable + public String getParameterName() { + final Value collectionValue = getUnsortedCollectionValue(); + return collectionValue instanceof ParameterObjectValue + ? ((ParameterObjectValue)collectionValue).getParameterName() + : null; + } + + /** + * Returns the elements of the literal list this source explodes, in the order the source produces them, if it + * explodes a literal list. + * @return the elements or {@code null} + */ + @Nullable + @SuppressWarnings("unchecked") + public List getLiteralValues() { + final Value collectionValue = getUnsortedCollectionValue(); + if (collectionValue instanceof LiteralValue && + ((LiteralValue)collectionValue).getLiteralValue() instanceof List) { + final List elements = (List)((LiteralValue)collectionValue).getLiteralValue(); + return isSorted() ? ArraySortValue.sort(elements, isReverse()) : elements; + } + return null; + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + if (mode.getKind() == PlanHashKind.LEGACY) { + // not every collection value supports the legacy mode + return PlanHashable.objectsPlanHash(mode, baseHash(mode, OBJECT_PLAN_HASH_IN_EXPLODE_SOURCE), getComparison(), + isSorted(), isReverse()); + } + // the comparison does not hash its comparand, so hash the explode, which hashes the collection + return PlanHashable.objectsPlanHash(mode, baseHash(mode, OBJECT_PLAN_HASH_IN_EXPLODE_SOURCE), explodeValue); + } + + @Override + public boolean isSorted() { + return explodeValue.getCollectionValue() instanceof ArraySortValue; + } + + @Override + public boolean isReverse() { + final Value collectionValue = explodeValue.getCollectionValue(); + return collectionValue instanceof ArraySortValue && ((ArraySortValue)collectionValue).isReverse(); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain() { + // explain the unsorted collection, the suffix tells whether it is sorted + return ExplainTokensWithPrecedence.of( + new ExplainTokens().addNested(ExplainLevel.ALL_DETAILS, + new Comparisons.ValueComparison(Comparisons.Type.IN, getUnsortedCollectionValue()) + .explain().getExplainTokens(), "...").addNested(explainSuffix())); + } + + @Nonnull + @Override + protected List getValues(@Nullable final EvaluationContext context) { + return explodeValue.evalAsList(null, context == null ? EvaluationContext.empty() : context); + } + + @Nonnull + @Override + public String toString() { + return getBindingName() + " IN " + explodeValue; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final InExplodeSource inExplodeSource = (InExplodeSource)o; + return super.equals(inExplodeSource) && explodeValue.equals(inExplodeSource.explodeValue); + } + + @Override + public int hashCode() { + return Objects.hash(getComparison(), explodeValue); + } + + @Nonnull + @Override + public Type getResultType() { + return explodeValue.getResultType(); + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return explodeValue.getDynamicTypes(); + } + + @Nonnull + @Override + public PInExplodeSource toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PInExplodeSource.newBuilder() + .setSuper(toInComparandSourceProto(serializationContext)) + .setExplodeValue(explodeValue.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + protected PInSource toInSourceProto(@Nonnull final PlanSerializationContext serializationContext) { + return PInSource.newBuilder().setInExplodeSource(toProto(serializationContext)).build(); + } + + @Nonnull + public static InExplodeSource fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PInExplodeSource inExplodeSourceProto) { + return new InExplodeSource(serializationContext, inExplodeSourceProto); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PInExplodeSource.class; + } + + @Nonnull + @Override + public InExplodeSource fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PInExplodeSource inExplodeSourceProto) { + return InExplodeSource.fromProto(serializationContext, inExplodeSourceProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java index 85eedba1804..b9bec62110f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java @@ -49,6 +49,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.AbstractRelationalExpressionWithChildren; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.explain.ExplainPlanVisitor; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -202,6 +203,15 @@ public Value getResultValue() { return inner.getFlowedObjectValue(); } + @Nonnull + @Override + protected Set computeDynamicTypes() { + final ImmutableSet.Builder resultBuilder = ImmutableSet.builder(); + resultBuilder.addAll(super.computeDynamicTypes()); + getInSources().forEach(inSource -> resultBuilder.addAll(inSource.getDynamicTypes())); + return resultBuilder.build(); + } + @Nonnull @Override public Set computeCorrelatedTo() { diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 5cb54a5ac31..1abb38cdffd 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -183,6 +183,10 @@ message RangeCursorContinuation { optional int64 nextPosition = 1; } +message ExplodeCursorContinuation { + optional int32 nextPosition = 1; +} + message KeyValueCursorContinuation { optional bytes inner_continuation = 1; optional sfixed64 magic_number = 2; diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 52a61a69362..7fd5e0a0eb1 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -287,6 +287,8 @@ message PValue { PCardinalityValue cardinality_value = 63; PArrayAggValue array_agg_value = 64; PCurrentTimestampValue current_timestamp_value = 65; + PExplodeValue explode_value = 66; + PArraySortValue array_sort_value = 67; } // Removed types that are no longer supported reserved 38; @@ -605,6 +607,9 @@ message PLikeOperatorValue { message PLiteralValue { optional PType result_type = 1; optional PComparableObject value = 2; + // set instead of value if the literal is a list + repeated PComparableObject list_elements = 3; + optional bool is_list = 4; } message PNotValue { @@ -1409,10 +1414,20 @@ message PRangeValue { optional PValue step_child = 3; } +message PExplodeValue { + optional PValue collection_value = 1; + optional bool with_ordinality = 2; +} + message PArrayDistinctValue { optional PValue child_value = 1; } +message PArraySortValue { + optional PValue child_value = 1; + optional bool reverse = 2; +} + message PParameterObjectValue { optional string parameter_name = 1; optional PType result_type = 2; @@ -2057,6 +2072,7 @@ message PInSource { PSortedInParameterSource sorted_in_parameter_source = 4; PInValuesSource in_values_source = 5; PSortedInValuesSource sorted_in_values_source = 6; + PInExplodeSource in_explode_source = 7; } } @@ -2090,6 +2106,11 @@ message PSortedInValuesSource { optional bool reverse = 2; } +message PInExplodeSource { + optional PInComparandSource super = 1; + optional PValue explode_value = 2; +} + // // PRecordQueryInParameterJoinPlan // diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java new file mode 100644 index 00000000000..44fd69df9c6 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -0,0 +1,392 @@ +/* + * ExplodeValueTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.Bindings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.TestRecords4Proto; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.explain.DefaultExplainFormatter; +import com.apple.foundationdb.record.query.plan.explain.DefaultExplainSymbolMap; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; +import com.apple.test.BooleanSource; +import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mockito; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for {@link ExplodeValue}. + */ +class ExplodeValueTest { + + private static final CorrelationIdentifier RESTAURANT_ALIAS = CorrelationIdentifier.of("restaurant"); + + @Nonnull + private static final FDBRecordStoreBase STORE = mockStore(); + + @Nonnull + @SuppressWarnings("unchecked") + private static FDBRecordStoreBase mockStore() { + final FDBRecordStoreBase store = Mockito.mock(FDBRecordStoreBase.class); + Mockito.when(store.getExecutor()).thenReturn(ForkJoinPool.commonPool()); + return store; + } + + @Nonnull + private static PlanSerializationContext newSerializationContext() { + return new PlanSerializationContext(new DefaultPlanSerializationRegistry(), + PlanHashable.CURRENT_FOR_CONTINUATION); + } + + @Nonnull + private static TestRecords4Proto.RestaurantReview review(final long reviewer, final int rating) { + return TestRecords4Proto.RestaurantReview.newBuilder().setReviewer(reviewer).setRating(rating).build(); + } + + @Nonnull + private static Value reviewsOfRestaurant() { + return FieldValue.ofFieldName(QuantifiedObjectValue.of(RESTAURANT_ALIAS, + Type.Record.fromDescriptor(TestRecords4Proto.RestaurantRecord.getDescriptor())), "reviews"); + } + + @Nonnull + private static EvaluationContext contextFor(@Nonnull final ExplodeValue value) { + return EvaluationContext.forTypeRepository(TypeRepository.newBuilder().addAllTypes(value.getDynamicTypes()).build()); + } + + @Nonnull + private static EvaluationContext contextFor(@Nonnull final ExplodeValue value, + @Nonnull final TestRecords4Proto.RestaurantRecord restaurant) { + return contextFor(value).withBinding(Bindings.Internal.CORRELATION, RESTAURANT_ALIAS, + QueryResult.ofComputed(restaurant)); + } + + /** + * Drains the stream of an explode, returning each struct it produces as the list of its fields, that is + * {@code [element]}, or {@code [element, ordinal]} for an explode with ordinality. + */ + @Nonnull + private static List> explode(@Nonnull final ExplodeValue value, @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + final var structs = value.evalAsStream(STORE, context, continuation, executeProperties).asList().join(); + return structs.stream() + .map(result -> fieldsOf(Objects.requireNonNull(result.getDatum()))) + .collect(ImmutableList.toImmutableList()); + } + + /** + * Evaluates an explode eagerly, returning each struct it produces as the list of its fields. + */ + @Nonnull + private static List> explodeAsList(@Nonnull final ExplodeValue value, @Nonnull final EvaluationContext context) { + return value.evalAsList(null, context).stream() + .map(ExplodeValueTest::fieldsOf) + .collect(ImmutableList.toImmutableList()); + } + + @Nonnull + private static List fieldsOf(@Nonnull final Object struct) { + final var message = (Message)struct; + return message.getDescriptorForType().getFields().stream() + .map(message::getField) + .collect(ImmutableList.toImmutableList()); + } + + @Nonnull + private static List> explode(@Nonnull final ExplodeValue value, @Nonnull final EvaluationContext context) { + return explode(value, context, null, ExecuteProperties.SERIAL_EXECUTE); + } + + @Test + void resultTypeIsTheElement() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b"))); + Assertions.assertFalse(value.hasOrdinality()); + + final var fields = value.getResultType().getFields(); + Assertions.assertEquals(1, fields.size()); + Assertions.assertEquals(value.getElementType(), fields.get(0).getFieldType()); + Assertions.assertEquals("_0", fields.get(0).getFieldName()); + Assertions.assertTrue(value.getDynamicTypes().contains(value.getResultType())); + } + + @Test + void resultTypeWithOrdinalityIsTheElementAndItsOrdinal() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b")), true); + Assertions.assertTrue(value.hasOrdinality()); + + final var fields = value.getResultType().getFields(); + Assertions.assertEquals(2, fields.size()); + Assertions.assertEquals(value.getElementType(), fields.get(0).getFieldType()); + Assertions.assertEquals(Type.primitiveType(Type.TypeCode.INT, false), fields.get(1).getFieldType()); + // Both fields are declared anonymous, the way an explode WITH ORDINALITY declares its struct, and are so named + // by their position. + Assertions.assertEquals("_0", fields.get(0).getFieldName()); + Assertions.assertEquals("_1", fields.get(1).getFieldName()); + + // The struct type is created by the value, so it is one of its dynamic types. + Assertions.assertTrue(value.getDynamicTypes().contains(value.getResultType())); + } + + @Test + void producesEveryElement() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c"))); + Assertions.assertEquals(List.of(List.of("a"), List.of("b"), List.of("c")), + explode(value, contextFor(value))); + } + + @Test + void producesEveryElementWithItsZeroBasedOrdinal() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), true); + Assertions.assertEquals(List.of(List.of("a", 0), List.of("b", 1), List.of("c", 2)), + explode(value, contextFor(value))); + } + + @Test + void producesOrdinalsEvenWhenTheElementsAreOfTheOrdinalType() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of(7, 7, 9)), true); + Assertions.assertEquals(List.of(List.of(7, 0), List.of(7, 1), List.of(9, 2)), + explode(value, contextFor(value))); + } + + @Test + void explodesAnArrayOfMessagesOfACorrelatedRecord() { + final var value = new ExplodeValue(reviewsOfRestaurant(), true); + final var restaurant = TestRecords4Proto.RestaurantRecord.newBuilder() + .setRestNo(1L) + .addReviews(review(10L, 5)) + .addReviews(review(20L, 3)) + .build(); + + final var exploded = explode(value, contextFor(value, restaurant)); + Assertions.assertEquals(2, exploded.size()); + final var firstReview = (Message)exploded.get(0).get(0); + final var secondReview = (Message)exploded.get(1).get(0); + Assertions.assertEquals(10L, firstReview.getField(firstReview.getDescriptorForType().findFieldByName("reviewer"))); + Assertions.assertEquals(20L, secondReview.getField(secondReview.getDescriptorForType().findFieldByName("reviewer"))); + Assertions.assertEquals(List.of(0, 1), List.of(exploded.get(0).get(1), exploded.get(1).get(1))); + } + + @ParameterizedTest + @BooleanSource + void evaluatingAsAListProducesWhatTheStreamProduces(final boolean withOrdinality) { + final var literal = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), withOrdinality); + final var literalContext = contextFor(literal); + Assertions.assertEquals(explode(literal, literalContext), explodeAsList(literal, literalContext)); + + final var reviews = new ExplodeValue(reviewsOfRestaurant(), withOrdinality); + final var restaurant = TestRecords4Proto.RestaurantRecord.newBuilder() + .setRestNo(1L) + .addReviews(review(10L, 5)) + .addReviews(review(20L, 3)) + .build(); + // the same context for both, so that the messages of the reviews share their descriptor + final var reviewsContext = contextFor(reviews, restaurant); + Assertions.assertEquals(explode(reviews, reviewsContext), explodeAsList(reviews, reviewsContext)); + } + + @Test + void evaluatingAsAListOfAnEmptyOrNullArrayProducesNothing() { + final var empty = new ExplodeValue(new LiteralValue<>(new Type.Array(Type.primitiveType(Type.TypeCode.INT, false)), List.of()), true); + Assertions.assertEquals(List.of(), explodeAsList(empty, contextFor(empty))); + + final var nullArray = new ExplodeValue(new NullValue(new Type.Array(Type.primitiveType(Type.TypeCode.INT, false))), true); + Assertions.assertEquals(List.of(), explodeAsList(nullArray, contextFor(nullArray))); + } + + @ParameterizedTest + @BooleanSource + void theOrdinalsOfASortedCollectionFollowTheSortedOrder(final boolean reverse) { + final var value = new ExplodeValue(new ArraySortValue(LiteralValue.ofList(List.of(30, 10, 20)), reverse), true); + Assertions.assertEquals(reverse + ? List.of(List.of(30, 0), List.of(20, 1), List.of(10, 2)) + : List.of(List.of(10, 0), List.of(20, 1), List.of(30, 2)), + explodeAsList(value, contextFor(value))); + } + + @Test + void anEmptyOrNullArrayProducesNothing() { + final var value = new ExplodeValue(reviewsOfRestaurant()); + final var restaurantWithoutReviews = TestRecords4Proto.RestaurantRecord.newBuilder().setRestNo(1L).build(); + Assertions.assertEquals(List.of(), explode(value, contextFor(value, restaurantWithoutReviews))); + + final var nullArray = new ExplodeValue(new NullValue(new Type.Array(Type.primitiveType(Type.TypeCode.INT, false)))); + Assertions.assertEquals(List.of(), explode(nullArray, contextFor(nullArray))); + } + + @Test + void honorsSkipAndLimit() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c", "d")), true); + final var executeProperties = ExecuteProperties.newBuilder().setSkip(1).setReturnedRowLimit(2).build(); + Assertions.assertEquals(List.of(List.of("b", 1), List.of("c", 2)), + explode(value, contextFor(value), null, executeProperties)); + } + + @Test + void resumesFromAContinuation() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), true); + final var context = contextFor(value); + + final RecordCursorResult first = + value.evalAsStream(STORE, context, null, ExecuteProperties.SERIAL_EXECUTE).getNext(); + Assertions.assertTrue(first.hasNext()); + + // The rest of the stream picks up after the first element, ordinals included. + Assertions.assertEquals(List.of(List.of("b", 1), List.of("c", 2)), + explode(value, context, first.getContinuation().toBytes(), ExecuteProperties.SERIAL_EXECUTE)); + } + + @Test + void theContinuationOfTheLastElementResumesToNothing() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b"))); + final var context = contextFor(value); + final var cursor = value.evalAsStream(STORE, context, null, ExecuteProperties.SERIAL_EXECUTE); + cursor.getNext(); + final RecordCursorResult last = cursor.getNext(); + Assertions.assertTrue(last.hasNext()); + + // A continuation handed out with a value is never an end continuation, even after the last element. + Assertions.assertFalse(last.getContinuation().isEnd()); + Assertions.assertEquals(List.of(), + explode(value, context, last.getContinuation().toBytes(), ExecuteProperties.SERIAL_EXECUTE)); + Assertions.assertTrue(cursor.getNext().getContinuation().isEnd()); + } + + @Test + void rejectsAContinuationPastTheEndOfTheArray() { + final var longer = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c"))); + final var cursor = longer.evalAsStream(STORE, contextFor(longer), null, ExecuteProperties.SERIAL_EXECUTE); + cursor.getNext(); + cursor.getNext(); + final byte[] continuation = cursor.getNext().getContinuation().toBytes(); + + // Resuming the same position over an array that has since shrunk is an error. + final var shorter = new ExplodeValue(LiteralValue.ofList(List.of("a"))); + Assertions.assertThrows(RecordCoreException.class, + () -> shorter.evalAsStream(STORE, contextFor(shorter), continuation, ExecuteProperties.SERIAL_EXECUTE)); + } + + @Test + void rejectsAContinuationThatCannotBeParsed() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a"))); + Assertions.assertThrows(RecordCoreException.class, + () -> value.evalAsStream(STORE, contextFor(value), new byte[] {(byte)0xff, (byte)0xff}, + ExecuteProperties.SERIAL_EXECUTE)); + } + + @Test + void createsAStructOnlyOnceTheCursorReachesItsElement() { + final var structsCreated = new AtomicInteger(); + final var cursor = new ExplodeValue.Cursor(ForkJoinPool.commonPool(), List.of("a", "b", "c"), + (array, ordinal) -> { + structsCreated.incrementAndGet(); + return List.of(array.get(ordinal), ordinal); + }, null); + Assertions.assertEquals(0, structsCreated.get()); + + Assertions.assertEquals(List.of("a", 0), Objects.requireNonNull(cursor.getNext().get()).getDatum()); + Assertions.assertEquals(1, structsCreated.get()); + } + + @Test + void rejectsACollectionThatIsNotAnArray() { + Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(LiteralValue.ofScalar(1))); + } + + @Test + void cannotBeEvaluatedAsAScalar() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of(1, 2))); + Assertions.assertThrows(IllegalStateException.class, () -> value.eval(null, contextFor(value))); + } + + @Test + void equalityAndHashingFollowTheCollection() { + final var collection = LiteralValue.ofList(List.of(1, 2)); + final var value = new ExplodeValue(collection); + + Assertions.assertEquals(value, new ExplodeValue(LiteralValue.ofList(List.of(1, 2)))); + Assertions.assertEquals(value.hashCode(), new ExplodeValue(LiteralValue.ofList(List.of(1, 2))).hashCode()); + Assertions.assertNotEquals(value, new ExplodeValue(LiteralValue.ofList(List.of(1, 3)))); + Assertions.assertNotEquals(value.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), + new ExplodeValue(LiteralValue.ofList(List.of(1, 3))).planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + + Assertions.assertSame(value, value.withChildren(List.of(collection))); + final var otherCollection = LiteralValue.ofList(List.of(3, 4)); + Assertions.assertEquals(new ExplodeValue(otherCollection), value.withChildren(List.of(otherCollection))); + } + + @Test + void equalityAndHashingFollowTheOrdinality() { + final var collection = LiteralValue.ofList(List.of(1, 2)); + final var value = new ExplodeValue(collection); + final var withOrdinality = new ExplodeValue(collection, true); + + Assertions.assertNotEquals(value, withOrdinality); + Assertions.assertNotEquals(value.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), + withOrdinality.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + + // rebasing the collection keeps the ordinality + final var otherCollection = LiteralValue.ofList(List.of(3, 4)); + Assertions.assertEquals(new ExplodeValue(otherCollection, true), withOrdinality.withChildren(List.of(otherCollection))); + Assertions.assertEquals(new ExplodeValue(otherCollection, true), withOrdinality.withCollectionValue(otherCollection)); + } + + @Test + void explainsAsAnExplodeOfTheCollection() { + final var value = new ExplodeValue(LiteralValue.ofList(List.of(1, 2))); + Assertions.assertEquals("explode([1, 2])", + value.explain().getExplainTokens().render(DefaultExplainFormatter.create(DefaultExplainSymbolMap::new)).toString()); + } + + @ParameterizedTest + @BooleanSource + void protoRoundTrip(final boolean withOrdinality) { + final var value = new ExplodeValue(reviewsOfRestaurant(), withOrdinality); + final var valueProto = value.toValueProto(newSerializationContext()); + Assertions.assertTrue(valueProto.hasExplodeValue()); + + final var deserialized = Value.fromValueProto(newSerializationContext(), valueProto); + Assertions.assertInstanceOf(ExplodeValue.class, deserialized); + Assertions.assertEquals(value, deserialized); + Assertions.assertEquals(value.getResultType(), deserialized.getResultType()); + Assertions.assertEquals(withOrdinality, ((ExplodeValue)deserialized).hasOrdinality()); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java new file mode 100644 index 00000000000..64d569dbe11 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -0,0 +1,127 @@ +/* + * InExplodeSourceTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; +import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.stream.Stream; + +/** + * Tests for {@link InExplodeSource} and the values it is built from, in particular that they survive serialization, + * so that a plan using them can be executed by a server that did not plan it. + */ +class InExplodeSourceTest { + + @Nonnull + private static PlanSerializationContext newSerializationContext() { + return new PlanSerializationContext(new DefaultPlanSerializationRegistry(), + PlanHashable.CURRENT_FOR_CONTINUATION); + } + + @Nonnull + private static Value elements() { + return LiteralValue.ofList(List.of(30L, 10L, 20L)); + } + + @Nonnull + private static List elementsOf(@Nonnull final InExplodeSource source) { + final var context = EvaluationContext.forTypeRepository( + TypeRepository.newBuilder().addAllTypes(source.getDynamicTypes()).build()); + return source.getValues(context).stream() + .map(struct -> { + final var message = (Message)struct; + return message.getField(message.getDescriptorForType().getFields().get(0)); + }) + .collect(ImmutableList.toImmutableList()); + } + + @Nonnull + static Stream sources() { + return Stream.of( + Arguments.of(new InExplodeSource("in", new ExplodeValue(elements())), List.of(30L, 10L, 20L)), + Arguments.of(new InExplodeSource("in", new ExplodeValue(new ArraySortValue(elements(), false))), List.of(10L, 20L, 30L)), + Arguments.of(new InExplodeSource("in", new ExplodeValue(new ArraySortValue(elements(), true))), List.of(30L, 20L, 10L))); + } + + @ParameterizedTest + @MethodSource("sources") + void producesTheElementsOfTheExplode(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + Assertions.assertEquals(expected, elementsOf(source)); + } + + @ParameterizedTest + @MethodSource("sources") + void protoRoundTrip(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + final var inSourceProto = source.toInSourceProto(newSerializationContext()); + Assertions.assertTrue(inSourceProto.hasInExplodeSource()); + + final var deserialized = InSource.fromInSourceProto(newSerializationContext(), inSourceProto); + Assertions.assertInstanceOf(InExplodeSource.class, deserialized); + Assertions.assertEquals(source, deserialized); + Assertions.assertEquals(source.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), + deserialized.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + Assertions.assertEquals(source.isSorted(), deserialized.isSorted()); + Assertions.assertEquals(source.isReverse(), deserialized.isReverse()); + Assertions.assertEquals(expected, elementsOf((InExplodeSource)deserialized)); + } + + @ParameterizedTest + @MethodSource("sources") + void arraySortValueProtoRoundTrip(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + final var collectionValue = new ExplodeValue(new ArraySortValue(elements(), source.isReverse())).getCollectionValue(); + final var valueProto = collectionValue.toValueProto(newSerializationContext()); + Assertions.assertTrue(valueProto.hasArraySortValue()); + + final var deserialized = Value.fromValueProto(newSerializationContext(), valueProto); + Assertions.assertInstanceOf(ArraySortValue.class, deserialized); + Assertions.assertEquals(collectionValue, deserialized); + Assertions.assertEquals(source.isReverse(), ((ArraySortValue)deserialized).isReverse()); + } + + @Test + void listLiteralProtoRoundTrip() { + final var literal = elements(); + final var valueProto = literal.toValueProto(newSerializationContext()); + Assertions.assertTrue(valueProto.getLiteralValue().getIsList()); + + final var deserialized = Value.fromValueProto(newSerializationContext(), valueProto); + Assertions.assertEquals(literal, deserialized); + Assertions.assertEquals(literal.getResultType(), deserialized.getResultType()); + Assertions.assertEquals(List.of(30L, 10L, 20L), ((LiteralValue)deserialized).getLiteralValue()); + } +} From f3b4a6d981d3181da7a56ab30c99bbdf66a545ce Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 2 Nov 2026 14:47:35 +0000 Subject: [PATCH 02/16] improvements --- .../properties/CardinalitiesProperty.java | 6 +- .../plan/cascades/values/ExplodeValue.java | 69 ++----------------- .../query/plan/plans/InExplodeSource.java | 39 ++++++----- .../plan/plans/RecordQueryInUnionPlan.java | 3 +- .../src/main/proto/record_query_plan.proto | 3 +- .../cascades/values/ExplodeValueTest.java | 4 +- .../query/plan/plans/InExplodeSourceTest.java | 24 +++++++ 7 files changed, 60 insertions(+), 88 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index bf7618e9f72..0e5107a64fe 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -58,6 +58,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.plans.InComparandSource; +import com.apple.foundationdb.record.query.plan.plans.InExplodeSource; import com.apple.foundationdb.record.query.plan.plans.InParameterSource; import com.apple.foundationdb.record.query.plan.plans.InValuesSource; import com.apple.foundationdb.record.query.plan.plans.QueryPlan; @@ -442,7 +443,10 @@ public Cardinalities visitRecordQueryInUnionPlan(@Nonnull final RecordQueryInUni final var inSourcesCardinalitiesOptional = inSources.stream() .map(inSource -> { - if (inSource instanceof InParameterSource || inSource instanceof InComparandSource) { + if (inSource instanceof InExplodeSource && ((InExplodeSource)inSource).getLiteralValues() != null) { + final var size = Objects.requireNonNull(((InExplodeSource)inSource).getLiteralValues()).size(); + return new Cardinalities(Cardinality.ofCardinality(size), Cardinality.ofCardinality(size)); + } else if (inSource instanceof InParameterSource || inSource instanceof InComparandSource) { return Cardinalities.unknownMaxCardinality(); } else { Verify.verify(inSource instanceof InValuesSource); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 7f11d0949d6..be280682ccb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -84,51 +84,26 @@ public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Explode-Value"); - /** - * The name of the field of the struct an explode produces that holds the element. - */ @Nonnull public static final String ELEMENT_FIELD_NAME = "_0"; - /** - * The type of the ordinal. - */ @Nonnull private static final Type ORDINAL_TYPE = Type.primitiveType(Type.TypeCode.INT, false); - /** - * The collection value. Must evaluate to an array type. - */ @Nonnull private final Value collectionValue; - /** - * The element type of the collection. - */ @Nonnull private final Type elementType; - /** - * Whether the struct this explode produces carries the ordinal of the element. - */ private final boolean withOrdinality; - /** - * The alias the element being produced is bound to while its struct is evaluated. - */ @Nonnull private final CorrelationIdentifier elementAlias; - /** - * The alias the ordinal of the element being produced is bound to while its struct is evaluated. Unused if this - * explode is not with ordinality. - */ @Nonnull private final CorrelationIdentifier ordinalAlias; - /** - * The struct produced per element: a record constructor of the element and, if with ordinality, its ordinal. - */ @Nonnull private final RecordConstructorValue structValue; @@ -171,35 +146,6 @@ public ExplodeValue withCollectionValue(@Nonnull final Value newCollectionValue) return new ExplodeValue(newCollectionValue, withOrdinality); } - @Nonnull - public Type getElementType() { - return elementType; - } - - /** - * Returns the element of a struct produced by an explode. - * - * @param explodedValue a value of the {@linkplain #getResultType() result type} of an explode, typically the - * flowed object value of a quantifier ranging over the explode - * @return the element the struct carries - */ - @Nonnull - public static Value elementValue(@Nonnull final Value explodedValue) { - return FieldValue.ofOrdinalNumberAndFuseIfPossible(explodedValue, 0); - } - - /** - * Returns the 0-based ordinal of a struct produced by an explode {@linkplain #hasOrdinality() with ordinality}. - * - * @param explodedValue a value of the {@linkplain #getResultType() result type} of an explode with ordinality, - * typically the flowed object value of a quantifier ranging over the explode - * @return the ordinal the struct carries - */ - @Nonnull - public static Value ordinalValue(@Nonnull final Value explodedValue) { - return FieldValue.ofOrdinalNumberAndFuseIfPossible(explodedValue, 1); - } - @Nonnull @Override public Type.Record getResultType() { @@ -265,7 +211,6 @@ private Object structOf(@Nonnull final EvaluationContext context, @Nonnull final */ @Nonnull private Object coerceElement(@Nonnull final TypeRepository typeRepository, @Nonnull final Object element) { - final Type elementType = structValue.getResultType().getField(0).getFieldType(); if (elementType instanceof Type.Enum) { if (element instanceof ProtocolMessageEnum) { return ((ProtocolMessageEnum)element).getValueDescriptor(); @@ -405,23 +350,20 @@ public ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializat } /** - * A cursor over the elements of an array, producing a struct of each element and its ordinal. A struct is only - * created once the cursor reaches its element. The continuation records the position of the next element. + * A cursor over the elements of an array, producing a struct of each element and its ordinal. The continuation + * records the position of the next element. */ - public static class Cursor implements RecordCursor { + static class Cursor implements RecordCursor { @Nonnull private final Executor executor; @Nonnull private final List elements; - /** - * Creates the struct of the element at an ordinal of the array, and that ordinal. - */ @Nonnull private final BiFunction, Integer, Object> structCreator; - private int nextPosition; // position of the next element to return + private int nextPosition; private boolean closed = false; @@ -494,9 +436,6 @@ private static class Continuation implements RecordCursorContinuation { @Override public boolean isEnd() { - // A continuation returned alongside a value must not be an end continuation, per the contract of - // RecordCursorResult, even if the cursor knows there is nothing after that value. Exhaustion is - // signalled by the exhausted result, which carries the end continuation. return false; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 4a168991b4b..5e26da76116 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.planprotos.PInExplodeSource; import com.apple.foundationdb.record.planprotos.PInSource; import com.apple.foundationdb.record.query.expressions.Comparisons; @@ -33,7 +34,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; -import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; @@ -67,9 +67,9 @@ public class InExplodeSource extends InComparandSource { protected InExplodeSource(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PInExplodeSource inExplodeSourceProto) { - super(serializationContext, Objects.requireNonNull(inExplodeSourceProto.getSuper())); - this.explodeValue = (ExplodeValue)Value.fromValueProto(serializationContext, - Objects.requireNonNull(inExplodeSourceProto.getExplodeValue())); + this(Objects.requireNonNull(inExplodeSourceProto.getSuper()).getBindingName(), + (ExplodeValue)Value.fromValueProto(serializationContext, + Objects.requireNonNull(inExplodeSourceProto.getExplodeValue()))); } public InExplodeSource(@Nonnull final String bindingName, @Nonnull final ExplodeValue explodeValue) { @@ -88,18 +88,6 @@ private Value getUnsortedCollectionValue() { return collectionValue instanceof ArraySortValue ? ((ArraySortValue)collectionValue).getChild() : collectionValue; } - /** - * Returns the name of the parameter this source explodes, if it explodes a parameter. - * @return the parameter name or {@code null} - */ - @Nullable - public String getParameterName() { - final Value collectionValue = getUnsortedCollectionValue(); - return collectionValue instanceof ParameterObjectValue - ? ((ParameterObjectValue)collectionValue).getParameterName() - : null; - } - /** * Returns the elements of the literal list this source explodes, in the order the source produces them, if it * explodes a literal list. @@ -149,10 +137,25 @@ public ExplainTokensWithPrecedence explain() { .explain().getExplainTokens(), "...").addNested(explainSuffix())); } + @Override + protected int size(@Nonnull final EvaluationContext context) { + final Object collection = getUnsortedCollectionValue().eval(null, context); + return collection == null ? 0 : ((List)collection).size(); + } + + /** + * Returns the structs of the elements of the collection. Creating them needs their type, so the context has to + * provide a type repository containing the {@linkplain #getResultType() result type} of this source. + * @param context the evaluation context + * @return the structs of the elements of the collection + */ @Nonnull @Override protected List getValues(@Nullable final EvaluationContext context) { - return explodeValue.evalAsList(null, context == null ? EvaluationContext.empty() : context); + if (context == null) { + throw new RecordCoreException("values of an in-explode source need an evaluation context"); + } + return explodeValue.evalAsList(null, context); } @Nonnull @@ -194,7 +197,7 @@ public Set getDynamicTypes() { @Override public PInExplodeSource toProto(@Nonnull final PlanSerializationContext serializationContext) { return PInExplodeSource.newBuilder() - .setSuper(toInComparandSourceProto(serializationContext)) + .setSuper(toInSourceSuperProto(serializationContext)) .setExplodeValue(explodeValue.toValueProto(serializationContext)) .build(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java index b9bec62110f..98b5a2a0d90 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java @@ -331,7 +331,8 @@ public String toString() { public int getComplexity() { int complexity = getInnerPlan().getComplexity(); for (InSource values : inSources) { - if (values instanceof InValuesSource) { + if (values instanceof InValuesSource || + (values instanceof InExplodeSource && ((InExplodeSource)values).getLiteralValues() != null)) { complexity *= values.size(EvaluationContext.EMPTY); } } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 7fd5e0a0eb1..ba0a97af80c 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -2107,7 +2107,8 @@ message PSortedInValuesSource { } message PInExplodeSource { - optional PInComparandSource super = 1; + // the comparison of the source is derived from the explode, so it is not serialized + optional PInSource.Super super = 1; optional PValue explode_value = 2; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 44fd69df9c6..4263723c5e9 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -143,7 +143,7 @@ void resultTypeIsTheElement() { final var fields = value.getResultType().getFields(); Assertions.assertEquals(1, fields.size()); - Assertions.assertEquals(value.getElementType(), fields.get(0).getFieldType()); + Assertions.assertEquals(((Type.Array)value.getCollectionValue().getResultType()).getElementType(), fields.get(0).getFieldType()); Assertions.assertEquals("_0", fields.get(0).getFieldName()); Assertions.assertTrue(value.getDynamicTypes().contains(value.getResultType())); } @@ -155,7 +155,7 @@ void resultTypeWithOrdinalityIsTheElementAndItsOrdinal() { final var fields = value.getResultType().getFields(); Assertions.assertEquals(2, fields.size()); - Assertions.assertEquals(value.getElementType(), fields.get(0).getFieldType()); + Assertions.assertEquals(((Type.Array)value.getCollectionValue().getResultType()).getElementType(), fields.get(0).getFieldType()); Assertions.assertEquals(Type.primitiveType(Type.TypeCode.INT, false), fields.get(1).getFieldType()); // Both fields are declared anonymous, the way an explode WITH ORDINALITY declares its struct, and are so named // by their position. diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 64d569dbe11..23e776ad2e2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -23,10 +23,13 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.google.common.collect.ImmutableList; @@ -84,6 +87,27 @@ void producesTheElementsOfTheExplode(@Nonnull final InExplodeSource source, @Non Assertions.assertEquals(expected, elementsOf(source)); } + @ParameterizedTest + @MethodSource("sources") + void sizeCountsTheElementsWithoutTheirType(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + // the empty context has no type repository, so creating any struct would fail + Assertions.assertEquals(expected.size(), source.size(EvaluationContext.EMPTY)); + } + + @Test + void sizeOfANullCollectionIsZero() { + final var source = new InExplodeSource("in", + new ExplodeValue(ParameterObjectValue.of("p", new Type.Array(Type.primitiveType(Type.TypeCode.LONG, false))))); + Assertions.assertEquals(0, source.size(EvaluationContext.EMPTY.withBinding("p", null))); + Assertions.assertEquals(2, source.size(EvaluationContext.EMPTY.withBinding("p", List.of(1L, 2L)))); + } + + @Test + void valuesNeedAnEvaluationContext() { + final var source = new InExplodeSource("in", new ExplodeValue(elements())); + Assertions.assertThrows(RecordCoreException.class, source::getValues); + } + @ParameterizedTest @MethodSource("sources") void protoRoundTrip(@Nonnull final InExplodeSource source, @Nonnull final List expected) { From abc957526bb0bdc0e0eb5b087bef744e597d5e96 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 2 Nov 2026 15:11:08 +0000 Subject: [PATCH 03/16] Only sort arrays whose elements can be sorted An array sort compares its elements as Comparables, so reject element types whose representation is not, such as records, bytes and vectors, when it is created. --- .../plan/cascades/values/ArraySortValue.java | 16 +++++++++++++++- .../query/plan/cascades/values/ExplodeValue.java | 6 ------ .../query/plan/plans/InComparandSource.java | 6 +++--- .../record/query/plan/plans/InExplodeSource.java | 3 +-- .../query/plan/plans/InExplodeSourceTest.java | 11 +++++++++++ 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java index 0f6478348cb..46954fc5407 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java @@ -64,11 +64,25 @@ public class ArraySortValue extends AbstractValue implements ValueWithChild { private final boolean reverse; public ArraySortValue(@Nonnull final Value childValue, final boolean reverse) { - Verify.verify(Objects.requireNonNull(childValue.getResultType()).isArray()); + final Type childType = Objects.requireNonNull(childValue.getResultType()); + Verify.verify(childType.isArray()); + final Type elementType = ((Type.Array)childType).getElementType(); + Verify.verify(elementType == null || isSortable(elementType), "cannot sort elements of type %s", elementType); this.childValue = childValue; this.reverse = reverse; } + /** + * Whether elements of a type can be sorted, which needs their representation to be {@link Comparable}. + */ + private static boolean isSortable(@Nonnull final Type elementType) { + if (elementType.isEnum()) { + return true; + } + return elementType.isPrimitive() && + elementType.getTypeCode() != Type.TypeCode.BYTES && elementType.getTypeCode() != Type.TypeCode.VECTOR; + } + public boolean isReverse() { return reverse; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index be280682ccb..31aab17aaa7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -163,12 +163,6 @@ public RecordCursor evalAsStream(@Nonnull final .skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); } - /** - * Evaluates this explode eagerly, producing the structs {@link #evalAsStream} would stream. - * @param store the record store, may be {@code null} if the collection does not need one - * @param context the evaluation context - * @return the structs of the elements of the collection, in the order of the collection - */ @Nonnull public List evalAsList(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { final List elements = elementsOf(store, context); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java index 391d80dc038..af6fd9214be 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java @@ -37,7 +37,6 @@ import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.google.auto.service.AutoService; -import com.google.common.collect.ImmutableSet; import com.google.protobuf.Message; import javax.annotation.Nonnull; @@ -176,14 +175,15 @@ public static InComparandSource fromProto(@Nonnull final PlanSerializationContex @Nonnull @Override public Type getResultType() { - return Objects.requireNonNull(comparison.getValue()).getResultType(); + final Value comparandValue = comparison.getValue(); + return comparandValue == null ? super.getResultType() : comparandValue.getResultType(); } @Nonnull @Override public Set getDynamicTypes() { final Value comparandValue = comparison.getValue(); - return comparandValue == null ? ImmutableSet.of() : comparandValue.getDynamicTypes(); + return comparandValue == null ? super.getDynamicTypes() : comparandValue.getDynamicTypes(); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 5e26da76116..28e57d727fa 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -144,8 +144,7 @@ protected int size(@Nonnull final EvaluationContext context) { } /** - * Returns the structs of the elements of the collection. Creating them needs their type, so the context has to - * provide a type repository containing the {@linkplain #getResultType() result type} of this source. + * Returns the structs of the elements of the collection. * @param context the evaluation context * @return the structs of the elements of the collection */ diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 23e776ad2e2..ab13b7de813 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -32,6 +32,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; +import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; import org.junit.jupiter.api.Assertions; @@ -42,6 +43,7 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; /** @@ -137,6 +139,15 @@ void arraySortValueProtoRoundTrip(@Nonnull final InExplodeSource source, @Nonnul Assertions.assertEquals(source.isReverse(), ((ArraySortValue)deserialized).isReverse()); } + @Test + void arraySortValueRejectsElementsThatCannotBeSorted() { + final var records = new LiteralValue<>(new Type.Array(Type.Record.fromFields(List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG), Optional.of("a"))))), List.of()); + Assertions.assertThrows(VerifyException.class, () -> new ArraySortValue(records, false)); + final var bytes = new LiteralValue<>(new Type.Array(Type.primitiveType(Type.TypeCode.BYTES, false)), List.of()); + Assertions.assertThrows(VerifyException.class, () -> new ArraySortValue(bytes, false)); + } + @Test void listLiteralProtoRoundTrip() { final var literal = elements(); From a7bd902311527c49b65de1bed4b76110e8bf198e Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 2 Nov 2026 15:44:39 +0000 Subject: [PATCH 04/16] improvements --- .../query/plan/cascades/values/ExplodeValue.java | 4 ++++ .../query/plan/cascades/values/LiteralValue.java | 2 ++ .../record/query/plan/plans/InExplodeSource.java | 3 ++- .../src/main/proto/record_query_plan.proto | 2 +- .../query/plan/plans/InExplodeSourceTest.java | 15 +++++++++++++++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 31aab17aaa7..d0469eada56 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -346,6 +346,10 @@ public ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializat /** * A cursor over the elements of an array, producing a struct of each element and its ordinal. The continuation * records the position of the next element. + *

+ * The continuation is not tied to the array it came from. Resuming is only correct if the array evaluates to the + * same elements in the same order as it did when the continuation was created. + *

*/ static class Cursor implements RecordCursor { @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java index 830fec1c746..46da5243784 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java @@ -168,6 +168,8 @@ public PLiteralValue toProto(@Nonnull final PlanSerializationContext serializati // TODO there should be no punishment if the type is serialized alongside with the value final var resultTypeProto = resultType.isVector() ? resultType.toTypeProto(serializationContext) : null; if (value instanceof List) { + // the elements are serialized like the comparand of a list comparison, so they are limited to the same + // scalar types builder.setIsList(true); for (final Object element : (List)value) { builder.addListElements(PlanSerialization.valueObjectToProto(element)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 28e57d727fa..88c555433be 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -74,7 +74,8 @@ protected InExplodeSource(@Nonnull final PlanSerializationContext serializationC public InExplodeSource(@Nonnull final String bindingName, @Nonnull final ExplodeValue explodeValue) { super(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue.getCollectionValue())); - Verify.verify(!explodeValue.hasOrdinality() || !(explodeValue.getCollectionValue() instanceof ArraySortValue)); + Verify.verify(!explodeValue.hasOrdinality() || !(explodeValue.getCollectionValue() instanceof ArraySortValue), + "cannot sort the collection of an explode with ordinality"); this.explodeValue = explodeValue; } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index ba0a97af80c..86cd5b076af 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -607,7 +607,7 @@ message PLikeOperatorValue { message PLiteralValue { optional PType result_type = 1; optional PComparableObject value = 2; - // set instead of value if the literal is a list + // set instead of value if the literal is a list; readers that do not know these fields read such a literal as null repeated PComparableObject list_elements = 3; optional bool is_list = 4; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index ab13b7de813..58f315bc4b6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -148,6 +148,13 @@ void arraySortValueRejectsElementsThatCannotBeSorted() { Assertions.assertThrows(VerifyException.class, () -> new ArraySortValue(bytes, false)); } + @Test + void sortedCollectionOfAnExplodeWithOrdinalityIsRejected() { + final var explodeValue = new ExplodeValue(new ArraySortValue(elements(), false), true); + final var exception = Assertions.assertThrows(VerifyException.class, () -> new InExplodeSource("in", explodeValue)); + Assertions.assertEquals("cannot sort the collection of an explode with ordinality", exception.getMessage()); + } + @Test void listLiteralProtoRoundTrip() { final var literal = elements(); @@ -159,4 +166,12 @@ void listLiteralProtoRoundTrip() { Assertions.assertEquals(literal.getResultType(), deserialized.getResultType()); Assertions.assertEquals(List.of(30L, 10L, 20L), ((LiteralValue)deserialized).getLiteralValue()); } + + @Test + void listLiteralWithNestedListCannotBeSerialized() { + final var literal = LiteralValue.ofList(List.of(List.of(1L, 2L))); + final var exception = Assertions.assertThrows(RecordCoreException.class, + () -> literal.toValueProto(newSerializationContext())); + Assertions.assertTrue(exception.getMessage().startsWith("Unsupported value type"), exception.getMessage()); + } } From 6c31cc28cddabb8db50301a836cd43aba6422f4f Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 2 Nov 2026 17:30:49 +0000 Subject: [PATCH 05/16] Use main's Javadoc paragraph style in ExplodeValue and InExplodeSource --- .../query/plan/cascades/values/ExplodeValue.java | 10 ++++------ .../record/query/plan/plans/InExplodeSource.java | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index d0469eada56..56ea400c7ed 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -75,10 +75,9 @@ * first anonymous field is the element. If the explode is {@linkplain #hasOrdinality() with ordinality}, the struct has * a second anonymous field that holds the 0-based ordinal of the element within the array, akin to SQL's * {@code UNNEST ... WITH ORDINALITY}. - *

- * An explode without ordinality does not expose the positions of the elements, so its collection can be replaced by + * + *

An explode without ordinality does not expose the positions of the elements, so its collection can be replaced by * any permutation of it, for example by a sorted one. - *

*/ @API(API.Status.EXPERIMENTAL) public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { @@ -346,10 +345,9 @@ public ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializat /** * A cursor over the elements of an array, producing a struct of each element and its ordinal. The continuation * records the position of the next element. - *

- * The continuation is not tied to the array it came from. Resuming is only correct if the array evaluates to the + * + *

The continuation is not tied to the array it came from. Resuming is only correct if the array evaluates to the * same elements in the same order as it did when the continuation was created. - *

*/ static class Cursor implements RecordCursor { @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 88c555433be..b41e2b68ea1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -51,11 +51,10 @@ * Source of values for an "IN" query that is planned from a quantifier ranging over an {@link ExplodeValue}. Like * the explode itself, it binds the struct of each element of the collection, so the correlated inner plan can read * the element and, if any, the ordinal the same way it would read them from the explode. - *

- * The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection would + * + *

The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection would * renumber the ordinals, which is why only the collections of explodes that are not * {@linkplain ExplodeValue#hasOrdinality() with ordinality} may be sorted. - *

*/ @API(API.Status.INTERNAL) public class InExplodeSource extends InComparandSource { From e3c824ef8ab1a9c9b99bf71e785f2329c1392b43 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Tue, 3 Nov 2026 16:21:55 +0000 Subject: [PATCH 06/16] Drop the redundant comment in InExplodeSource.explain() --- .../foundationdb/record/query/plan/plans/InExplodeSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index b41e2b68ea1..72b0587d68d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -130,7 +130,6 @@ public boolean isReverse() { @Nonnull @Override public ExplainTokensWithPrecedence explain() { - // explain the unsorted collection, the suffix tells whether it is sorted return ExplainTokensWithPrecedence.of( new ExplainTokens().addNested(ExplainLevel.ALL_DETAILS, new Comparisons.ValueComparison(Comparisons.Type.IN, getUnsortedCollectionValue()) From 353178515108090a5e324e7c933ef7a126f9c41d Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 4 Nov 2026 09:33:58 +0000 Subject: [PATCH 07/16] Derive the struct of an explode from its collection The derivations of a table function over an ExplodeValue are a struct of the first element of the collection (and an ordinal, if the explode has one), as for the explode plan before, rather than the first struct the explode flows. That keeps the correlations of the collection and the shape of the struct the explode flows. --- .../properties/DerivationsProperty.java | 13 +++++++++ .../cascades/values/ExplodeValueTest.java | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index b3ea80006bb..da8d0997669 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValue; import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultValue; import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; @@ -394,6 +395,18 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl @Override public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { final var streamingValue = tableFunctionPlan.getValue(); + if (streamingValue instanceof final ExplodeValue explodeValue) { + // Use the first element of the collection as a representative for "some" element, and wrap it in the + // struct the explode flows, so that accesses of the element are written against the struct. + final var elementType = Objects.requireNonNull(((Type.Array)explodeValue.getCollectionValue().getResultType()).getElementType()); + final var columns = ImmutableList.>builder(); + columns.add(Column.unnamedOf(new FirstOrDefaultValue(explodeValue.getCollectionValue(), new ThrowsValue(elementType)))); + if (explodeValue.hasOrdinality()) { + columns.add(Column.unnamedOf(new ThrowsValue(Type.primitiveType(Type.TypeCode.INT, false)))); + } + final var representatives = ImmutableList.of(RecordConstructorValue.ofColumns(columns.build(), false)); + return new Derivations(representatives, representatives); + } final var elementType = streamingValue.getResultType(); final var values = ImmutableList.of(new FirstOrDefaultStreamingValue(streamingValue, new ThrowsValue(elementType))); return new Derivations(values, values); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 4263723c5e9..80cc724b94b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -30,11 +30,13 @@ import com.apple.foundationdb.record.TestRecords4Proto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.properties.DerivationsProperty; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.explain.DefaultExplainFormatter; import com.apple.foundationdb.record.query.plan.explain.DefaultExplainSymbolMap; import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.apple.test.BooleanSource; import com.google.common.base.VerifyException; @@ -49,6 +51,7 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicInteger; @@ -376,6 +379,30 @@ void explainsAsAnExplodeOfTheCollection() { value.explain().getExplainTokens().render(DefaultExplainFormatter.create(DefaultExplainSymbolMap::new)).toString()); } + @ParameterizedTest + @BooleanSource + void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final boolean withOrdinality) { + final var sourceAlias = CorrelationIdentifier.of("source"); + final var arrayType = new Type.Array(false, Type.primitiveType(Type.TypeCode.INT, false)); + final var sourceType = Type.Record.fromFields(List.of(Type.Record.Field.of(arrayType, Optional.of("arr")))); + final Value collectionValue = FieldValue.ofFieldName(QuantifiedObjectValue.of(sourceAlias, sourceType), "arr"); + final var explodeValue = new ExplodeValue(collectionValue, withOrdinality); + + final var derivations = new DerivationsProperty.DerivationsVisitor() + .visitTableFunctionPlan(new RecordQueryTableFunctionPlan(explodeValue)); + + Assertions.assertEquals(1, derivations.getResultValues().size()); + final Value derived = derivations.getResultValues().get(0); + // the representative of some struct of the explode is still correlated to what the collection is correlated to + Assertions.assertTrue(derived.getCorrelatedTo().contains(sourceAlias)); + // and it has the shape of the struct the explode flows: the first element of the collection, and the ordinal + Assertions.assertEquals(explodeValue.getResultType(), derived.getResultType()); + Assertions.assertTrue(derived instanceof RecordConstructorValue); + final List columns = ImmutableList.copyOf(derived.getChildren()); + Assertions.assertEquals(withOrdinality ? 2 : 1, columns.size()); + Assertions.assertTrue(columns.get(0) instanceof FirstOrDefaultValue); + } + @ParameterizedTest @BooleanSource void protoRoundTrip(final boolean withOrdinality) { From d40d04dffae06a87bc65d3f6b36b2e1db8e5ea8c Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 00:34:13 +0000 Subject: [PATCH 08/16] Drop the special cases for explodes of literal lists The planner no longer builds explodes over literal lists or parameters, so the list serialization of literals, the literal and parameter accessors of the in-explode source, and the coercion of literal elements are not needed. --- .../properties/CardinalitiesProperty.java | 6 +-- .../plan/cascades/values/ExplodeValue.java | 37 +------------ .../plan/cascades/values/LiteralValue.java | 52 +++---------------- .../query/plan/plans/InExplodeSource.java | 18 ------- .../plan/plans/RecordQueryInUnionPlan.java | 3 +- .../src/main/proto/record_query_plan.proto | 3 -- .../query/plan/plans/InExplodeSourceTest.java | 24 ++------- 7 files changed, 13 insertions(+), 130 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index 0e5107a64fe..bf7618e9f72 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -58,7 +58,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.plans.InComparandSource; -import com.apple.foundationdb.record.query.plan.plans.InExplodeSource; import com.apple.foundationdb.record.query.plan.plans.InParameterSource; import com.apple.foundationdb.record.query.plan.plans.InValuesSource; import com.apple.foundationdb.record.query.plan.plans.QueryPlan; @@ -443,10 +442,7 @@ public Cardinalities visitRecordQueryInUnionPlan(@Nonnull final RecordQueryInUni final var inSourcesCardinalitiesOptional = inSources.stream() .map(inSource -> { - if (inSource instanceof InExplodeSource && ((InExplodeSource)inSource).getLiteralValues() != null) { - final var size = Objects.requireNonNull(((InExplodeSource)inSource).getLiteralValues()).size(); - return new Cardinalities(Cardinality.ofCardinality(size), Cardinality.ofCardinality(size)); - } else if (inSource instanceof InParameterSource || inSource instanceof InComparandSource) { + if (inSource instanceof InParameterSource || inSource instanceof InComparandSource) { return Cardinalities.unknownMaxCardinality(); } else { Verify.verify(inSource instanceof InValuesSource); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 56ea400c7ed..b133e7ecf82 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -45,11 +45,9 @@ import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.properties.CardinalitiesProperty; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.plans.QueryResult; -import com.apple.foundationdb.record.util.ProtoUtils; import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.google.auto.service.AutoService; import com.google.common.base.Verify; @@ -58,7 +56,6 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; -import com.google.protobuf.ProtocolMessageEnum; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -188,7 +185,7 @@ private Object structAt(@Nonnull final EvaluationContext context, @Nonnull final private Object structOf(@Nonnull final EvaluationContext context, @Nonnull final Object element, final int ordinal) { EvaluationContext elementContext = context .withBinding(Bindings.Internal.CORRELATION, elementAlias, - QueryResult.ofComputed(coerceElement(context.getTypeRepository(), element))); + QueryResult.ofComputed(element)); if (withOrdinality) { elementContext = elementContext.withBinding(Bindings.Internal.CORRELATION, ordinalAlias, QueryResult.ofComputed(ordinal)); @@ -196,44 +193,12 @@ private Object structOf(@Nonnull final EvaluationContext context, @Nonnull final return Objects.requireNonNull(structValue.eval(null, elementContext)); } - /** - * Coerces an element to the representation of the element type in the struct. Elements of literal lists, such - * as the lists of IN predicates, are plain Java objects that do not necessarily use that representation: numbers - * may be narrower than the element type, and enums are Java enums or {@link ProtoUtils.DynamicEnum}s rather than - * enum value descriptors. - */ - @Nonnull - private Object coerceElement(@Nonnull final TypeRepository typeRepository, @Nonnull final Object element) { - if (elementType instanceof Type.Enum) { - if (element instanceof ProtocolMessageEnum) { - return ((ProtocolMessageEnum)element).getValueDescriptor(); - } - if (element instanceof ProtoUtils.DynamicEnum) { - return Objects.requireNonNull(typeRepository.getEnumValue(typeRepository.getProtoTypeName(elementType), - ((ProtoUtils.DynamicEnum)element).getName())); - } - return element; - } - if (element instanceof final Number number && elementType.isPrimitive()) { - return switch (elementType.getTypeCode()) { - case LONG -> element instanceof Long ? element : (Object)number.longValue(); - case DOUBLE -> element instanceof Double ? element : (Object)number.doubleValue(); - case FLOAT -> element instanceof Float ? element : (Object)number.floatValue(); - default -> element; - }; - } - return element; - } - @Nonnull @Override public CardinalitiesProperty.Cardinalities getCardinalities() { final int size; if (collectionValue instanceof AbstractArrayConstructorValue) { size = Iterables.size(collectionValue.getChildren()); - } else if (collectionValue instanceof LiteralValue && - ((LiteralValue)collectionValue).getLiteralValue() instanceof List) { - size = ((List)((LiteralValue)collectionValue).getLiteralValue()).size(); } else { return CardinalitiesProperty.Cardinalities.unknownMaxCardinality(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java index 46da5243784..dda428e3524 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/LiteralValue.java @@ -28,7 +28,6 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializable; import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.planprotos.PComparableObject; import com.apple.foundationdb.record.planprotos.PLiteralValue; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; @@ -47,7 +46,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -120,31 +118,13 @@ public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { return LeafValue.super.equalsWithoutChildren(other) .filter(ignored -> { final LiteralValue that = (LiteralValue)other; - return literalsEqual(value, that.value); + if (value == null && that.value == null) { + return true; + } + return Boolean.TRUE.equals(Comparisons.evalComparison(Comparisons.Type.EQUALS, value, that.value)); }); } - private static boolean literalsEqual(@Nullable final Object value, @Nullable final Object otherValue) { - if (value == null && otherValue == null) { - return true; - } - if (value instanceof List && otherValue instanceof List) { - // compare element-wise, so that elements that do not round-trip as themselves, e.g. enums, still compare - final List elements = (List)value; - final List otherElements = (List)otherValue; - if (elements.size() != otherElements.size()) { - return false; - } - for (int i = 0; i < elements.size(); i++) { - if (!literalsEqual(elements.get(i), otherElements.get(i))) { - return false; - } - } - return true; - } - return Boolean.TRUE.equals(Comparisons.evalComparison(Comparisons.Type.EQUALS, value, otherValue)); - } - @Override public int hashCodeWithoutChildren() { return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, value); @@ -167,16 +147,7 @@ public PLiteralValue toProto(@Nonnull final PlanSerializationContext serializati final var builder = PLiteralValue.newBuilder(); // TODO there should be no punishment if the type is serialized alongside with the value final var resultTypeProto = resultType.isVector() ? resultType.toTypeProto(serializationContext) : null; - if (value instanceof List) { - // the elements are serialized like the comparand of a list comparison, so they are limited to the same - // scalar types - builder.setIsList(true); - for (final Object element : (List)value) { - builder.addListElements(PlanSerialization.valueObjectToProto(element)); - } - } else { - builder.setValue(PlanSerialization.valueObjectToProto(value, resultTypeProto)); - } + builder.setValue(PlanSerialization.valueObjectToProto(value, resultTypeProto)); builder.setResultType(resultType.toTypeProto(serializationContext)); return builder.build(); } @@ -194,17 +165,8 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization @Nonnull public static LiteralValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PLiteralValue literalValueProto) { - final Object value; - if (literalValueProto.getIsList()) { - final List elements = new ArrayList<>(literalValueProto.getListElementsCount()); - for (final PComparableObject elementProto : literalValueProto.getListElementsList()) { - elements.add(PlanSerialization.protoToValueObject(elementProto)); - } - value = elements; - } else { - value = PlanSerialization.protoToValueObject(literalValueProto.getValue()); - } - return new LiteralValue<>(Type.fromTypeProto(serializationContext, literalValueProto.getResultType()), (T)value); + return new LiteralValue<>(Type.fromTypeProto(serializationContext, literalValueProto.getResultType()), + (T)PlanSerialization.protoToValueObject(literalValueProto.getValue())); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 72b0587d68d..235ed9eadce 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -33,7 +33,6 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; -import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; @@ -88,23 +87,6 @@ private Value getUnsortedCollectionValue() { return collectionValue instanceof ArraySortValue ? ((ArraySortValue)collectionValue).getChild() : collectionValue; } - /** - * Returns the elements of the literal list this source explodes, in the order the source produces them, if it - * explodes a literal list. - * @return the elements or {@code null} - */ - @Nullable - @SuppressWarnings("unchecked") - public List getLiteralValues() { - final Value collectionValue = getUnsortedCollectionValue(); - if (collectionValue instanceof LiteralValue && - ((LiteralValue)collectionValue).getLiteralValue() instanceof List) { - final List elements = (List)((LiteralValue)collectionValue).getLiteralValue(); - return isSorted() ? ArraySortValue.sort(elements, isReverse()) : elements; - } - return null; - } - @Override public int planHash(@Nonnull final PlanHashMode mode) { if (mode.getKind() == PlanHashKind.LEGACY) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java index 98b5a2a0d90..b9bec62110f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInUnionPlan.java @@ -331,8 +331,7 @@ public String toString() { public int getComplexity() { int complexity = getInnerPlan().getComplexity(); for (InSource values : inSources) { - if (values instanceof InValuesSource || - (values instanceof InExplodeSource && ((InExplodeSource)values).getLiteralValues() != null)) { + if (values instanceof InValuesSource) { complexity *= values.size(EvaluationContext.EMPTY); } } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 86cd5b076af..e4c1daa8fe8 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -607,9 +607,6 @@ message PLikeOperatorValue { message PLiteralValue { optional PType result_type = 1; optional PComparableObject value = 2; - // set instead of value if the literal is a list; readers that do not know these fields read such a literal as null - repeated PComparableObject list_elements = 3; - optional bool is_list = 4; } message PNotValue { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 58f315bc4b6..8e957362d91 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; @@ -60,7 +61,8 @@ private static PlanSerializationContext newSerializationContext() { @Nonnull private static Value elements() { - return LiteralValue.ofList(List.of(30L, 10L, 20L)); + return AbstractArrayConstructorValue.LightArrayConstructorValue.of( + List.of(LiteralValue.ofScalar(30L), LiteralValue.ofScalar(10L), LiteralValue.ofScalar(20L))); } @Nonnull @@ -154,24 +156,4 @@ void sortedCollectionOfAnExplodeWithOrdinalityIsRejected() { final var exception = Assertions.assertThrows(VerifyException.class, () -> new InExplodeSource("in", explodeValue)); Assertions.assertEquals("cannot sort the collection of an explode with ordinality", exception.getMessage()); } - - @Test - void listLiteralProtoRoundTrip() { - final var literal = elements(); - final var valueProto = literal.toValueProto(newSerializationContext()); - Assertions.assertTrue(valueProto.getLiteralValue().getIsList()); - - final var deserialized = Value.fromValueProto(newSerializationContext(), valueProto); - Assertions.assertEquals(literal, deserialized); - Assertions.assertEquals(literal.getResultType(), deserialized.getResultType()); - Assertions.assertEquals(List.of(30L, 10L, 20L), ((LiteralValue)deserialized).getLiteralValue()); - } - - @Test - void listLiteralWithNestedListCannotBeSerialized() { - final var literal = LiteralValue.ofList(List.of(List.of(1L, 2L))); - final var exception = Assertions.assertThrows(RecordCoreException.class, - () -> literal.toValueProto(newSerializationContext())); - Assertions.assertTrue(exception.getMessage().startsWith("Unsupported value type"), exception.getMessage()); - } } From 9ba75c79a5cec0e979f4d128a89691047e111ffa Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 14:22:03 +0000 Subject: [PATCH 09/16] update --- .../properties/DerivationsProperty.java | 4 +- .../plan/cascades/values/ExplodeValue.java | 25 +++++------ .../query/plan/plans/InExplodeSource.java | 8 ++-- .../cascades/values/ExplodeValueTest.java | 41 ++++++------------- 4 files changed, 27 insertions(+), 51 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index da8d0997669..ea94ca7e86b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -401,9 +401,7 @@ public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctio final var elementType = Objects.requireNonNull(((Type.Array)explodeValue.getCollectionValue().getResultType()).getElementType()); final var columns = ImmutableList.>builder(); columns.add(Column.unnamedOf(new FirstOrDefaultValue(explodeValue.getCollectionValue(), new ThrowsValue(elementType)))); - if (explodeValue.hasOrdinality()) { - columns.add(Column.unnamedOf(new ThrowsValue(Type.primitiveType(Type.TypeCode.INT, false)))); - } + columns.add(Column.unnamedOf(new ThrowsValue(Type.primitiveType(Type.TypeCode.INT, false)))); final var representatives = ImmutableList.of(RecordConstructorValue.ofColumns(columns.build(), false)); return new Derivations(representatives, representatives); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index b133e7ecf82..0cb9e052dda 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -68,13 +68,14 @@ import java.util.stream.IntStream; /** - * A {@link StreamingValue} that iterates over the elements of an array. For every element it produces a struct whose - * first anonymous field is the element. If the explode is {@linkplain #hasOrdinality() with ordinality}, the struct has - * a second anonymous field that holds the 0-based ordinal of the element within the array, akin to SQL's + * A {@link StreamingValue} that iterates over the elements of an array. For every element it produces a struct of two + * anonymous fields: the element, and the 0-based ordinal of the element within the array, akin to SQL's * {@code UNNEST ... WITH ORDINALITY}. * - *

An explode without ordinality does not expose the positions of the elements, so its collection can be replaced by - * any permutation of it, for example by a sorted one. + *

The struct has the same shape whether or not the explode is {@linkplain #hasOrdinality() with ordinality}. The + * ordinality only states whether the ordinals are observed. The ordinals of an explode without ordinality are not, so + * its collection can be replaced by any permutation of it, for example by a sorted one, even though that changes the + * ordinals it produces. */ @API(API.Status.EXPERIMENTAL) public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { @@ -116,9 +117,7 @@ public ExplodeValue(@Nonnull final Value collectionValue, final boolean withOrdi this.ordinalAlias = CorrelationIdentifier.uniqueId(); final ImmutableList.Builder> columnsBuilder = ImmutableList.builder(); columnsBuilder.add(Column.unnamedOf(QuantifiedObjectValue.of(elementAlias, elementType))); - if (withOrdinality) { - columnsBuilder.add(Column.unnamedOf(QuantifiedObjectValue.of(ordinalAlias, ORDINAL_TYPE))); - } + columnsBuilder.add(Column.unnamedOf(QuantifiedObjectValue.of(ordinalAlias, ORDINAL_TYPE))); this.structValue = RecordConstructorValue.ofColumns(columnsBuilder.build(), false); Verify.verify(ELEMENT_FIELD_NAME.equals(structValue.getResultType().getField(0).getFieldName())); } @@ -183,13 +182,9 @@ private Object structAt(@Nonnull final EvaluationContext context, @Nonnull final @Nonnull private Object structOf(@Nonnull final EvaluationContext context, @Nonnull final Object element, final int ordinal) { - EvaluationContext elementContext = context - .withBinding(Bindings.Internal.CORRELATION, elementAlias, - QueryResult.ofComputed(element)); - if (withOrdinality) { - elementContext = elementContext.withBinding(Bindings.Internal.CORRELATION, ordinalAlias, - QueryResult.ofComputed(ordinal)); - } + final EvaluationContext elementContext = context + .withBinding(Bindings.Internal.CORRELATION, elementAlias, QueryResult.ofComputed(element)) + .withBinding(Bindings.Internal.CORRELATION, ordinalAlias, QueryResult.ofComputed(ordinal)); return Objects.requireNonNull(structValue.eval(null, elementContext)); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 235ed9eadce..b3d3afffb52 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -49,11 +49,11 @@ /** * Source of values for an "IN" query that is planned from a quantifier ranging over an {@link ExplodeValue}. Like * the explode itself, it binds the struct of each element of the collection, so the correlated inner plan can read - * the element and, if any, the ordinal the same way it would read them from the explode. + * the element and the ordinal the same way it would read them from the explode. * - *

The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection would - * renumber the ordinals, which is why only the collections of explodes that are not - * {@linkplain ExplodeValue#hasOrdinality() with ordinality} may be sorted. + *

The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection + * renumbers the ordinals, which is why only the collections of explodes whose ordinals are not observed, that is that + * are not {@linkplain ExplodeValue#hasOrdinality() with ordinality}, may be sorted. */ @API(API.Status.INTERNAL) public class InExplodeSource extends InComparandSource { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 80cc724b94b..1c5b14a56c7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -104,7 +104,7 @@ private static EvaluationContext contextFor(@Nonnull final ExplodeValue value, /** * Drains the stream of an explode, returning each struct it produces as the list of its fields, that is - * {@code [element]}, or {@code [element, ordinal]} for an explode with ordinality. + * {@code [element, ordinal]}. */ @Nonnull private static List> explode(@Nonnull final ExplodeValue value, @Nonnull final EvaluationContext context, @@ -139,29 +139,18 @@ private static List> explode(@Nonnull final ExplodeValue value, @No return explode(value, context, null, ExecuteProperties.SERIAL_EXECUTE); } - @Test - void resultTypeIsTheElement() { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b"))); - Assertions.assertFalse(value.hasOrdinality()); - - final var fields = value.getResultType().getFields(); - Assertions.assertEquals(1, fields.size()); - Assertions.assertEquals(((Type.Array)value.getCollectionValue().getResultType()).getElementType(), fields.get(0).getFieldType()); - Assertions.assertEquals("_0", fields.get(0).getFieldName()); - Assertions.assertTrue(value.getDynamicTypes().contains(value.getResultType())); - } - - @Test - void resultTypeWithOrdinalityIsTheElementAndItsOrdinal() { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b")), true); - Assertions.assertTrue(value.hasOrdinality()); + @ParameterizedTest + @BooleanSource + void resultTypeIsTheElementAndItsOrdinal(final boolean withOrdinality) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b")), withOrdinality); + Assertions.assertEquals(withOrdinality, value.hasOrdinality()); final var fields = value.getResultType().getFields(); Assertions.assertEquals(2, fields.size()); Assertions.assertEquals(((Type.Array)value.getCollectionValue().getResultType()).getElementType(), fields.get(0).getFieldType()); Assertions.assertEquals(Type.primitiveType(Type.TypeCode.INT, false), fields.get(1).getFieldType()); // Both fields are declared anonymous, the way an explode WITH ORDINALITY declares its struct, and are so named - // by their position. + // by their position. The ordinality does not change the shape. Assertions.assertEquals("_0", fields.get(0).getFieldName()); Assertions.assertEquals("_1", fields.get(1).getFieldName()); @@ -169,16 +158,10 @@ void resultTypeWithOrdinalityIsTheElementAndItsOrdinal() { Assertions.assertTrue(value.getDynamicTypes().contains(value.getResultType())); } - @Test - void producesEveryElement() { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c"))); - Assertions.assertEquals(List.of(List.of("a"), List.of("b"), List.of("c")), - explode(value, contextFor(value))); - } - - @Test - void producesEveryElementWithItsZeroBasedOrdinal() { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), true); + @ParameterizedTest + @BooleanSource + void producesEveryElementWithItsZeroBasedOrdinal(final boolean withOrdinality) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), withOrdinality); Assertions.assertEquals(List.of(List.of("a", 0), List.of("b", 1), List.of("c", 2)), explode(value, contextFor(value))); } @@ -399,7 +382,7 @@ void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final b Assertions.assertEquals(explodeValue.getResultType(), derived.getResultType()); Assertions.assertTrue(derived instanceof RecordConstructorValue); final List columns = ImmutableList.copyOf(derived.getChildren()); - Assertions.assertEquals(withOrdinality ? 2 : 1, columns.size()); + Assertions.assertEquals(2, columns.size()); Assertions.assertTrue(columns.get(0) instanceof FirstOrDefaultValue); } From 95fda276ce3496163641742c73e4a8ac2c67a7ce Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 15:12:25 +0000 Subject: [PATCH 10/16] Rename the ordinality of an explode to preserveOrder Every explode flows the ordinal now, so the flag no longer says whether there is one. It says whether the planner must keep the collection in its order, which is what keeps the ordinals meaningful. --- .../plan/cascades/values/ExplodeValue.java | 33 ++++++++------- .../query/plan/plans/InExplodeSource.java | 8 ++-- .../src/main/proto/record_query_plan.proto | 2 +- .../cascades/values/ExplodeValueTest.java | 42 +++++++++---------- .../query/plan/plans/InExplodeSourceTest.java | 4 +- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 0cb9e052dda..599c5aa9b9e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -72,10 +72,11 @@ * anonymous fields: the element, and the 0-based ordinal of the element within the array, akin to SQL's * {@code UNNEST ... WITH ORDINALITY}. * - *

The struct has the same shape whether or not the explode is {@linkplain #hasOrdinality() with ordinality}. The - * ordinality only states whether the ordinals are observed. The ordinals of an explode without ordinality are not, so - * its collection can be replaced by any permutation of it, for example by a sorted one, even though that changes the - * ordinals it produces. + *

The struct has the same shape whether or not the explode {@linkplain #preservesOrder() preserves the order} of + * its collection. An explode that preserves the order produces the elements in the order of the array, so each ordinal + * is the position of its element in the array, and the planner must not reorder the collection. Any other explode + * does not observe its ordinals, so its collection can be replaced by any permutation of it, for example by a sorted + * one, even though that renumbers the ordinals it produces. */ @API(API.Status.EXPERIMENTAL) public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { @@ -93,7 +94,7 @@ public class ExplodeValue extends AbstractValue implements StreamingValue, Creat @Nonnull private final Type elementType; - private final boolean withOrdinality; + private final boolean preserveOrder; @Nonnull private final CorrelationIdentifier elementAlias; @@ -108,10 +109,10 @@ public ExplodeValue(@Nonnull final Value collectionValue) { this(collectionValue, false); } - public ExplodeValue(@Nonnull final Value collectionValue, final boolean withOrdinality) { + public ExplodeValue(@Nonnull final Value collectionValue, final boolean preserveOrder) { Verify.verify(collectionValue.getResultType().isArray()); this.collectionValue = collectionValue; - this.withOrdinality = withOrdinality; + this.preserveOrder = preserveOrder; this.elementType = Objects.requireNonNull(((Type.Array)collectionValue.getResultType()).getElementType()); this.elementAlias = CorrelationIdentifier.uniqueId(); this.ordinalAlias = CorrelationIdentifier.uniqueId(); @@ -127,18 +128,18 @@ public Value getCollectionValue() { return collectionValue; } - public boolean hasOrdinality() { - return withOrdinality; + public boolean preservesOrder() { + return preserveOrder; } /** - * Returns an explode of a different collection that is with ordinality if and only if this one is. + * Returns an explode of a different collection that preserves its order if and only if this one does. * @param newCollectionValue the new collection * @return an explode of the new collection */ @Nonnull public ExplodeValue withCollectionValue(@Nonnull final Value newCollectionValue) { - return new ExplodeValue(newCollectionValue, withOrdinality); + return new ExplodeValue(newCollectionValue, preserveOrder); } @Nonnull @@ -234,19 +235,19 @@ public ExplainTokensWithPrecedence explain(@Nonnull final Iterable withOrdinality == ((ExplodeValue)other).hasOrdinality()); + .filter(ignored -> preserveOrder == ((ExplodeValue)other).preservesOrder()); } @SpotBugsSuppressWarnings("EQ_UNUSUAL") @@ -266,7 +267,7 @@ public int hashCode() { public PExplodeValue toProto(@Nonnull final PlanSerializationContext serializationContext) { return PExplodeValue.newBuilder() .setCollectionValue(collectionValue.toValueProto(serializationContext)) - .setWithOrdinality(withOrdinality) + .setPreserveOrder(preserveOrder) .build(); } @@ -280,7 +281,7 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization public static ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PExplodeValue explodeValueProto) { return new ExplodeValue(Value.fromValueProto(serializationContext, - Objects.requireNonNull(explodeValueProto.getCollectionValue())), explodeValueProto.getWithOrdinality()); + Objects.requireNonNull(explodeValueProto.getCollectionValue())), explodeValueProto.getPreserveOrder()); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index b3d3afffb52..9eac69ff4ef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -52,8 +52,8 @@ * the element and the ordinal the same way it would read them from the explode. * *

The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection - * renumbers the ordinals, which is why only the collections of explodes whose ordinals are not observed, that is that - * are not {@linkplain ExplodeValue#hasOrdinality() with ordinality}, may be sorted. + * renumbers the ordinals, which is why only the collections of explodes that do not + * {@linkplain ExplodeValue#preservesOrder() preserve their order} may be sorted. */ @API(API.Status.INTERNAL) public class InExplodeSource extends InComparandSource { @@ -72,8 +72,8 @@ protected InExplodeSource(@Nonnull final PlanSerializationContext serializationC public InExplodeSource(@Nonnull final String bindingName, @Nonnull final ExplodeValue explodeValue) { super(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue.getCollectionValue())); - Verify.verify(!explodeValue.hasOrdinality() || !(explodeValue.getCollectionValue() instanceof ArraySortValue), - "cannot sort the collection of an explode with ordinality"); + Verify.verify(!explodeValue.preservesOrder() || !(explodeValue.getCollectionValue() instanceof ArraySortValue), + "cannot sort the collection of an explode that preserves its order"); this.explodeValue = explodeValue; } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index e4c1daa8fe8..040e7d1741c 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1413,7 +1413,7 @@ message PRangeValue { message PExplodeValue { optional PValue collection_value = 1; - optional bool with_ordinality = 2; + optional bool preserve_order = 2; } message PArrayDistinctValue { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 1c5b14a56c7..6508addf9c2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -141,16 +141,16 @@ private static List> explode(@Nonnull final ExplodeValue value, @No @ParameterizedTest @BooleanSource - void resultTypeIsTheElementAndItsOrdinal(final boolean withOrdinality) { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b")), withOrdinality); - Assertions.assertEquals(withOrdinality, value.hasOrdinality()); + void resultTypeIsTheElementAndItsOrdinal(final boolean preserveOrder) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b")), preserveOrder); + Assertions.assertEquals(preserveOrder, value.preservesOrder()); final var fields = value.getResultType().getFields(); Assertions.assertEquals(2, fields.size()); Assertions.assertEquals(((Type.Array)value.getCollectionValue().getResultType()).getElementType(), fields.get(0).getFieldType()); Assertions.assertEquals(Type.primitiveType(Type.TypeCode.INT, false), fields.get(1).getFieldType()); // Both fields are declared anonymous, the way an explode WITH ORDINALITY declares its struct, and are so named - // by their position. The ordinality does not change the shape. + // by their position. Preserving the order does not change the shape. Assertions.assertEquals("_0", fields.get(0).getFieldName()); Assertions.assertEquals("_1", fields.get(1).getFieldName()); @@ -160,8 +160,8 @@ void resultTypeIsTheElementAndItsOrdinal(final boolean withOrdinality) { @ParameterizedTest @BooleanSource - void producesEveryElementWithItsZeroBasedOrdinal(final boolean withOrdinality) { - final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), withOrdinality); + void producesEveryElementWithItsZeroBasedOrdinal(final boolean preserveOrder) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), preserveOrder); Assertions.assertEquals(List.of(List.of("a", 0), List.of("b", 1), List.of("c", 2)), explode(value, contextFor(value))); } @@ -193,12 +193,12 @@ void explodesAnArrayOfMessagesOfACorrelatedRecord() { @ParameterizedTest @BooleanSource - void evaluatingAsAListProducesWhatTheStreamProduces(final boolean withOrdinality) { - final var literal = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), withOrdinality); + void evaluatingAsAListProducesWhatTheStreamProduces(final boolean preserveOrder) { + final var literal = new ExplodeValue(LiteralValue.ofList(List.of("a", "b", "c")), preserveOrder); final var literalContext = contextFor(literal); Assertions.assertEquals(explode(literal, literalContext), explodeAsList(literal, literalContext)); - final var reviews = new ExplodeValue(reviewsOfRestaurant(), withOrdinality); + final var reviews = new ExplodeValue(reviewsOfRestaurant(), preserveOrder); final var restaurant = TestRecords4Proto.RestaurantRecord.newBuilder() .setRestNo(1L) .addReviews(review(10L, 5)) @@ -340,19 +340,19 @@ void equalityAndHashingFollowTheCollection() { } @Test - void equalityAndHashingFollowTheOrdinality() { + void equalityAndHashingFollowWhetherTheOrderIsPreserved() { final var collection = LiteralValue.ofList(List.of(1, 2)); final var value = new ExplodeValue(collection); - final var withOrdinality = new ExplodeValue(collection, true); + final var ordered = new ExplodeValue(collection, true); - Assertions.assertNotEquals(value, withOrdinality); + Assertions.assertNotEquals(value, ordered); Assertions.assertNotEquals(value.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), - withOrdinality.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + ordered.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); - // rebasing the collection keeps the ordinality + // rebasing the collection keeps preserving the order final var otherCollection = LiteralValue.ofList(List.of(3, 4)); - Assertions.assertEquals(new ExplodeValue(otherCollection, true), withOrdinality.withChildren(List.of(otherCollection))); - Assertions.assertEquals(new ExplodeValue(otherCollection, true), withOrdinality.withCollectionValue(otherCollection)); + Assertions.assertEquals(new ExplodeValue(otherCollection, true), ordered.withChildren(List.of(otherCollection))); + Assertions.assertEquals(new ExplodeValue(otherCollection, true), ordered.withCollectionValue(otherCollection)); } @Test @@ -364,12 +364,12 @@ void explainsAsAnExplodeOfTheCollection() { @ParameterizedTest @BooleanSource - void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final boolean withOrdinality) { + void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final boolean preserveOrder) { final var sourceAlias = CorrelationIdentifier.of("source"); final var arrayType = new Type.Array(false, Type.primitiveType(Type.TypeCode.INT, false)); final var sourceType = Type.Record.fromFields(List.of(Type.Record.Field.of(arrayType, Optional.of("arr")))); final Value collectionValue = FieldValue.ofFieldName(QuantifiedObjectValue.of(sourceAlias, sourceType), "arr"); - final var explodeValue = new ExplodeValue(collectionValue, withOrdinality); + final var explodeValue = new ExplodeValue(collectionValue, preserveOrder); final var derivations = new DerivationsProperty.DerivationsVisitor() .visitTableFunctionPlan(new RecordQueryTableFunctionPlan(explodeValue)); @@ -388,8 +388,8 @@ void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final b @ParameterizedTest @BooleanSource - void protoRoundTrip(final boolean withOrdinality) { - final var value = new ExplodeValue(reviewsOfRestaurant(), withOrdinality); + void protoRoundTrip(final boolean preserveOrder) { + final var value = new ExplodeValue(reviewsOfRestaurant(), preserveOrder); final var valueProto = value.toValueProto(newSerializationContext()); Assertions.assertTrue(valueProto.hasExplodeValue()); @@ -397,6 +397,6 @@ void protoRoundTrip(final boolean withOrdinality) { Assertions.assertInstanceOf(ExplodeValue.class, deserialized); Assertions.assertEquals(value, deserialized); Assertions.assertEquals(value.getResultType(), deserialized.getResultType()); - Assertions.assertEquals(withOrdinality, ((ExplodeValue)deserialized).hasOrdinality()); + Assertions.assertEquals(preserveOrder, ((ExplodeValue)deserialized).preservesOrder()); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 8e957362d91..7980e6ffdf0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -151,9 +151,9 @@ void arraySortValueRejectsElementsThatCannotBeSorted() { } @Test - void sortedCollectionOfAnExplodeWithOrdinalityIsRejected() { + void sortedCollectionOfAnExplodeThatPreservesOrderIsRejected() { final var explodeValue = new ExplodeValue(new ArraySortValue(elements(), false), true); final var exception = Assertions.assertThrows(VerifyException.class, () -> new InExplodeSource("in", explodeValue)); - Assertions.assertEquals("cannot sort the collection of an explode with ordinality", exception.getMessage()); + Assertions.assertEquals("cannot sort the collection of an explode that preserves its order", exception.getMessage()); } } From f82b59cad8ff96e46cdf1f7e685327aa9e74c752 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 15:36:46 +0000 Subject: [PATCH 11/16] Add an empty line before the block tags of the explode Javadoc --- .../record/query/plan/cascades/values/ArraySortValue.java | 1 + .../record/query/plan/cascades/values/ExplodeValue.java | 1 + .../foundationdb/record/query/plan/plans/InExplodeSource.java | 2 ++ 3 files changed, 4 insertions(+) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java index 46954fc5407..a9a316e20ef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java @@ -126,6 +126,7 @@ public Object eval(@Nullable final FDBRecordStoreBase sto /** * Sorts a list of comparable elements. + * * @param elements the elements to sort * @param reverse whether to sort in descending order * @return a sorted copy of the elements diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 599c5aa9b9e..00d3ce59f86 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -134,6 +134,7 @@ public boolean preservesOrder() { /** * Returns an explode of a different collection that preserves its order if and only if this one does. + * * @param newCollectionValue the new collection * @return an explode of the new collection */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 9eac69ff4ef..8cc7755ec2e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -79,6 +79,7 @@ public InExplodeSource(@Nonnull final String bindingName, @Nonnull final Explode /** * Returns the collection the explode of this source ranges over before it is sorted. + * * @return the unsorted collection */ @Nonnull @@ -126,6 +127,7 @@ protected int size(@Nonnull final EvaluationContext context) { /** * Returns the structs of the elements of the collection. + * * @param context the evaluation context * @return the structs of the elements of the collection */ From e6c64bf131349632deec12ee6b88c2e19d04f55a Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 15:47:52 +0000 Subject: [PATCH 12/16] Document what preserveOrder means for an explode --- .../record/query/plan/cascades/values/ExplodeValue.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 00d3ce59f86..9106fe61d28 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -94,6 +94,12 @@ public class ExplodeValue extends AbstractValue implements StreamingValue, Creat @Nonnull private final Type elementType; + /** + * Whether the planner must keep the collection in its order. If set, every ordinal this explode produces is the + * position of its element in the array, so the collection must not be replaced by a permutation of it, for example + * by a sorted one. If not set, the ordinals are not observed and the collection may be reordered. Either way the + * explode flows the same {@code (element, ordinal)} struct. + */ private final boolean preserveOrder; @Nonnull From ceb301214de1e06a9293f1bc540bb44673a99464 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 18:22:18 +0000 Subject: [PATCH 13/16] Revert the comparand-less fallbacks in InComparandSource They existed for the explodes of literal lists and parameters, which are gone. InExplodeSource overrides both methods itself, so every remaining InComparandSource has a comparand value. --- .../record/query/plan/plans/InComparandSource.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java index af6fd9214be..5b14744151b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InComparandSource.java @@ -32,7 +32,6 @@ import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; @@ -175,15 +174,13 @@ public static InComparandSource fromProto(@Nonnull final PlanSerializationContex @Nonnull @Override public Type getResultType() { - final Value comparandValue = comparison.getValue(); - return comparandValue == null ? super.getResultType() : comparandValue.getResultType(); + return Objects.requireNonNull(comparison.getValue()).getResultType(); } @Nonnull @Override public Set getDynamicTypes() { - final Value comparandValue = comparison.getValue(); - return comparandValue == null ? super.getDynamicTypes() : comparandValue.getDynamicTypes(); + return Objects.requireNonNull(comparison.getValue()).getDynamicTypes(); } /** From 5215323c0980e87f30bc54edd6bbbaf32aa17469 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 18:47:30 +0000 Subject: [PATCH 14/16] Make the sort order a property of the explode rather than of its collection ExplodeValue gains sorted and reverse flags, and sorts the elements itself. An IN source over a sorted explode no longer has to recognise an ArraySortValue as the collection, which would need digging up once anything else wraps the sort, so ArraySortValue is removed. --- .../plan/cascades/values/ArraySortValue.java | 222 ------------------ .../plan/cascades/values/ExplodeValue.java | 109 ++++++++- .../query/plan/plans/InExplodeSource.java | 29 +-- .../src/main/proto/record_query_plan.proto | 8 +- .../cascades/values/ExplodeValueTest.java | 67 +++++- .../query/plan/plans/InExplodeSourceTest.java | 36 +-- 6 files changed, 174 insertions(+), 297 deletions(-) delete mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java deleted file mode 100644 index a9a316e20ef..00000000000 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArraySortValue.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * ArraySortValue.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.query.plan.cascades.values; - -import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; -import com.apple.foundationdb.record.EvaluationContext; -import com.apple.foundationdb.record.ObjectPlanHash; -import com.apple.foundationdb.record.PlanDeserializer; -import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.planprotos.PArraySortValue; -import com.apple.foundationdb.record.planprotos.PValue; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; -import com.apple.foundationdb.record.query.plan.cascades.AliasMap; -import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; -import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; -import com.google.auto.service.AutoService; -import com.google.common.base.Verify; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.Message; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.function.Supplier; - -/** - * A value that returns a copy of its child array {@link Value} with its elements sorted in ascending or, if reverse, - * descending order. The elements must be comparable with each other. - */ -@API(API.Status.EXPERIMENTAL) -public class ArraySortValue extends AbstractValue implements ValueWithChild { - private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Array-Sort-Value"); - - @SuppressWarnings("unchecked") - private static final Comparator ELEMENT_COMPARATOR = Comparator.comparing(Comparable.class::cast); - - @Nonnull - private final Value childValue; - private final boolean reverse; - - public ArraySortValue(@Nonnull final Value childValue, final boolean reverse) { - final Type childType = Objects.requireNonNull(childValue.getResultType()); - Verify.verify(childType.isArray()); - final Type elementType = ((Type.Array)childType).getElementType(); - Verify.verify(elementType == null || isSortable(elementType), "cannot sort elements of type %s", elementType); - this.childValue = childValue; - this.reverse = reverse; - } - - /** - * Whether elements of a type can be sorted, which needs their representation to be {@link Comparable}. - */ - private static boolean isSortable(@Nonnull final Type elementType) { - if (elementType.isEnum()) { - return true; - } - return elementType.isPrimitive() && - elementType.getTypeCode() != Type.TypeCode.BYTES && elementType.getTypeCode() != Type.TypeCode.VECTOR; - } - - public boolean isReverse() { - return reverse; - } - - @Nonnull - @Override - public List computeChildren() { - return ImmutableList.of(childValue); - } - - @Nonnull - @Override - public Value getChild() { - return childValue; - } - - @Nonnull - @Override - @SuppressWarnings("PMD.CompareObjectsWithEquals") - public ValueWithChild withNewChild(@Nonnull final Value rebasedChild) { - if (getChild() == rebasedChild) { - return this; - } - return new ArraySortValue(rebasedChild, reverse); - } - - @Nonnull - @Override - public Type getResultType() { - return childValue.getResultType(); - } - - @Override - public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - final var childResult = childValue.eval(store, context); - if (childResult == null) { - return null; - } - return sort((List)childResult, reverse); - } - - /** - * Sorts a list of comparable elements. - * - * @param elements the elements to sort - * @param reverse whether to sort in descending order - * @return a sorted copy of the elements - */ - @Nonnull - public static List sort(@Nonnull final List elements, final boolean reverse) { - final List sorted = new ArrayList<>(elements); - sorted.sort(reverse ? ELEMENT_COMPARATOR.reversed() : ELEMENT_COMPARATOR); - return sorted; - } - - @Override - public int hashCodeWithoutChildren() { - return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH, reverse); - } - - @Override - public int planHash(@Nonnull final PlanHashMode mode) { - return PlanHashable.objectsPlanHash(mode, BASE_HASH, childValue, reverse); - } - - @Nonnull - @Override - public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { - return super.equalsWithoutChildren(other) - .filter(ignored -> reverse == ((ArraySortValue)other).isReverse()); - } - - @Nonnull - @Override - public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { - final var explainTokens = new ExplainTokens().addFunctionCall("arraySort", - Value.explainFunctionArguments(explainSuppliers)); - if (reverse) { - explainTokens.addWhitespace().addKeyword("DESC"); - } - return ExplainTokensWithPrecedence.of(explainTokens); - } - - @Override - public int hashCode() { - return semanticHashCode(); - } - - @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") - @SpotBugsSuppressWarnings("EQ_UNUSUAL") - @Override - public boolean equals(final Object other) { - return semanticEquals(other, AliasMap.emptyMap()); - } - - @Nonnull - @Override - public PArraySortValue toProto(@Nonnull final PlanSerializationContext serializationContext) { - return PArraySortValue.newBuilder() - .setChildValue(childValue.toValueProto(serializationContext)) - .setReverse(reverse) - .build(); - } - - @Nonnull - @Override - public PValue toValueProto(@Nonnull PlanSerializationContext serializationContext) { - return PValue.newBuilder().setArraySortValue(toProto(serializationContext)).build(); - } - - @Nonnull - public static ArraySortValue fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PArraySortValue arraySortValueProto) { - return new ArraySortValue( - Value.fromValueProto(serializationContext, Objects.requireNonNull(arraySortValueProto.getChildValue())), - arraySortValueProto.getReverse()); - } - - /** - * Deserializer. - */ - @AutoService(PlanDeserializer.class) - public static class Deserializer implements PlanDeserializer { - @Nonnull - @Override - public Class getProtoMessageClass() { - return PArraySortValue.class; - } - - @Nonnull - @Override - public ArraySortValue fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PArraySortValue arraySortValueProto) { - return ArraySortValue.fromProto(serializationContext, arraySortValueProto); - } - } -} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index 9106fe61d28..e89eb683b1c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -59,6 +59,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; @@ -77,6 +79,11 @@ * is the position of its element in the array, and the planner must not reorder the collection. Any other explode * does not observe its ordinals, so its collection can be replaced by any permutation of it, for example by a sorted * one, even though that renumbers the ordinals it produces. + * + *

An explode that is {@linkplain #isSorted() sorted} produces the elements of its collection in ascending order, or + * in descending order if it is also {@linkplain #isReverse() reverse}, and numbers the ordinals in that order. The + * order is a property of the explode rather than of its collection, so it does not have to be recovered from whatever + * computes the collection. Only an explode that does not preserve the order of its collection can be sorted. */ @API(API.Status.EXPERIMENTAL) public class ExplodeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { @@ -88,6 +95,10 @@ public class ExplodeValue extends AbstractValue implements StreamingValue, Creat @Nonnull private static final Type ORDINAL_TYPE = Type.primitiveType(Type.TypeCode.INT, false); + @Nonnull + @SuppressWarnings("unchecked") + private static final Comparator ELEMENT_COMPARATOR = Comparator.comparing(Comparable.class::cast); + @Nonnull private final Value collectionValue; @@ -102,6 +113,16 @@ public class ExplodeValue extends AbstractValue implements StreamingValue, Creat */ private final boolean preserveOrder; + /** + * Whether the explode produces the elements of its collection sorted rather than in the order of the array. + */ + private final boolean sorted; + + /** + * Whether a sorted explode sorts in descending rather than ascending order. + */ + private final boolean reverse; + @Nonnull private final CorrelationIdentifier elementAlias; @@ -116,10 +137,20 @@ public ExplodeValue(@Nonnull final Value collectionValue) { } public ExplodeValue(@Nonnull final Value collectionValue, final boolean preserveOrder) { + this(collectionValue, preserveOrder, false, false); + } + + public ExplodeValue(@Nonnull final Value collectionValue, final boolean preserveOrder, final boolean sorted, + final boolean reverse) { Verify.verify(collectionValue.getResultType().isArray()); + Verify.verify(!preserveOrder || !sorted, "cannot sort an explode that preserves its order"); + Verify.verify(sorted || !reverse, "cannot reverse an explode that is not sorted"); this.collectionValue = collectionValue; this.preserveOrder = preserveOrder; + this.sorted = sorted; + this.reverse = reverse; this.elementType = Objects.requireNonNull(((Type.Array)collectionValue.getResultType()).getElementType()); + Verify.verify(!sorted || isSortable(elementType), "cannot sort elements of type %s", elementType); this.elementAlias = CorrelationIdentifier.uniqueId(); this.ordinalAlias = CorrelationIdentifier.uniqueId(); final ImmutableList.Builder> columnsBuilder = ImmutableList.builder(); @@ -138,15 +169,49 @@ public boolean preservesOrder() { return preserveOrder; } + public boolean isSorted() { + return sorted; + } + + public boolean isReverse() { + return reverse; + } + /** - * Returns an explode of a different collection that preserves its order if and only if this one does. + * Whether elements of a type can be sorted, which needs their representation to be {@link Comparable}. + * + * @param elementType the type of the elements + * @return {@code true} if an explode of an array of such elements can be sorted + */ + public static boolean isSortable(@Nonnull final Type elementType) { + if (elementType.isEnum()) { + return true; + } + return elementType.isPrimitive() && + elementType.getTypeCode() != Type.TypeCode.BYTES && elementType.getTypeCode() != Type.TypeCode.VECTOR; + } + + /** + * Returns an explode of a different collection that preserves its order, and is sorted, if and only if this one + * does and is. * * @param newCollectionValue the new collection * @return an explode of the new collection */ @Nonnull public ExplodeValue withCollectionValue(@Nonnull final Value newCollectionValue) { - return new ExplodeValue(newCollectionValue, preserveOrder); + return new ExplodeValue(newCollectionValue, preserveOrder, sorted, reverse); + } + + /** + * Returns an explode of the same collection that produces its elements sorted. + * + * @param reverse whether to sort in descending order + * @return a sorted explode of the same collection + */ + @Nonnull + public ExplodeValue sorted(final boolean reverse) { + return new ExplodeValue(collectionValue, preserveOrder, true, reverse); } @Nonnull @@ -175,12 +240,21 @@ public List evalAsList(@Nullable final FDBRecordStoreBase store, @Non } /** - * Evaluates the collection. A {@code null} collection has no elements. + * Evaluates the collection, and sorts it if the explode is sorted. A {@code null} collection has no elements. */ @Nonnull private List elementsOf(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { final Object result = collectionValue.eval(store, context); - return result == null ? List.of() : (List)result; + if (result == null) { + return List.of(); + } + final List elements = (List)result; + if (!sorted) { + return elements; + } + final List sortedElements = new ArrayList<>(elements); + sortedElements.sort(reverse ? ELEMENT_COMPARATOR.reversed() : ELEMENT_COMPARATOR); + return sortedElements; } @Nonnull @@ -236,25 +310,37 @@ public Value withChildren(final Iterable newChildren) { @Nonnull @Override public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { - return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("explode", - Iterables.getOnlyElement(explainSuppliers).get().getExplainTokens())); + final var explainTokens = new ExplainTokens().addFunctionCall("explode", + Iterables.getOnlyElement(explainSuppliers).get().getExplainTokens()); + if (sorted) { + explainTokens.addWhitespace().addKeyword("SORTED"); + if (reverse) { + explainTokens.addWhitespace().addKeyword("DESC"); + } + } + return ExplainTokensWithPrecedence.of(explainTokens); } @Override public int hashCodeWithoutChildren() { - return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH, preserveOrder); + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH, preserveOrder, sorted, + reverse); } @Override public int planHash(@Nonnull final PlanHashMode mode) { - return PlanHashable.objectsPlanHash(mode, BASE_HASH, collectionValue, preserveOrder); + return PlanHashable.objectsPlanHash(mode, BASE_HASH, collectionValue, preserveOrder, sorted, reverse); } @Nonnull @Override public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { return super.equalsWithoutChildren(other) - .filter(ignored -> preserveOrder == ((ExplodeValue)other).preservesOrder()); + .filter(ignored -> { + final var otherExplodeValue = (ExplodeValue)other; + return preserveOrder == otherExplodeValue.preservesOrder() && + sorted == otherExplodeValue.isSorted() && reverse == otherExplodeValue.isReverse(); + }); } @SpotBugsSuppressWarnings("EQ_UNUSUAL") @@ -275,6 +361,8 @@ public PExplodeValue toProto(@Nonnull final PlanSerializationContext serializati return PExplodeValue.newBuilder() .setCollectionValue(collectionValue.toValueProto(serializationContext)) .setPreserveOrder(preserveOrder) + .setSorted(sorted) + .setReverse(reverse) .build(); } @@ -288,7 +376,8 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization public static ExplodeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PExplodeValue explodeValueProto) { return new ExplodeValue(Value.fromValueProto(serializationContext, - Objects.requireNonNull(explodeValueProto.getCollectionValue())), explodeValueProto.getPreserveOrder()); + Objects.requireNonNull(explodeValueProto.getCollectionValue())), explodeValueProto.getPreserveOrder(), + explodeValueProto.getSorted(), explodeValueProto.getReverse()); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java index 8cc7755ec2e..56e164c619e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSource.java @@ -31,14 +31,12 @@ import com.apple.foundationdb.record.planprotos.PInSource; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.explain.ExplainLevel; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.google.auto.service.AutoService; -import com.google.common.base.Verify; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -51,9 +49,8 @@ * the explode itself, it binds the struct of each element of the collection, so the correlated inner plan can read * the element and the ordinal the same way it would read them from the explode. * - *

The source is sorted if the collection of the explode is an {@link ArraySortValue}. Sorting the collection - * renumbers the ordinals, which is why only the collections of explodes that do not - * {@linkplain ExplodeValue#preservesOrder() preserve their order} may be sorted. + *

The source is sorted if the explode is {@linkplain ExplodeValue#isSorted() sorted}, and reverse if the explode + * is {@linkplain ExplodeValue#isReverse() reverse}. */ @API(API.Status.INTERNAL) public class InExplodeSource extends InComparandSource { @@ -72,22 +69,9 @@ protected InExplodeSource(@Nonnull final PlanSerializationContext serializationC public InExplodeSource(@Nonnull final String bindingName, @Nonnull final ExplodeValue explodeValue) { super(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue.getCollectionValue())); - Verify.verify(!explodeValue.preservesOrder() || !(explodeValue.getCollectionValue() instanceof ArraySortValue), - "cannot sort the collection of an explode that preserves its order"); this.explodeValue = explodeValue; } - /** - * Returns the collection the explode of this source ranges over before it is sorted. - * - * @return the unsorted collection - */ - @Nonnull - private Value getUnsortedCollectionValue() { - final Value collectionValue = explodeValue.getCollectionValue(); - return collectionValue instanceof ArraySortValue ? ((ArraySortValue)collectionValue).getChild() : collectionValue; - } - @Override public int planHash(@Nonnull final PlanHashMode mode) { if (mode.getKind() == PlanHashKind.LEGACY) { @@ -101,13 +85,12 @@ public int planHash(@Nonnull final PlanHashMode mode) { @Override public boolean isSorted() { - return explodeValue.getCollectionValue() instanceof ArraySortValue; + return explodeValue.isSorted(); } @Override public boolean isReverse() { - final Value collectionValue = explodeValue.getCollectionValue(); - return collectionValue instanceof ArraySortValue && ((ArraySortValue)collectionValue).isReverse(); + return explodeValue.isReverse(); } @Nonnull @@ -115,13 +98,13 @@ public boolean isReverse() { public ExplainTokensWithPrecedence explain() { return ExplainTokensWithPrecedence.of( new ExplainTokens().addNested(ExplainLevel.ALL_DETAILS, - new Comparisons.ValueComparison(Comparisons.Type.IN, getUnsortedCollectionValue()) + new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue.getCollectionValue()) .explain().getExplainTokens(), "...").addNested(explainSuffix())); } @Override protected int size(@Nonnull final EvaluationContext context) { - final Object collection = getUnsortedCollectionValue().eval(null, context); + final Object collection = explodeValue.getCollectionValue().eval(null, context); return collection == null ? 0 : ((List)collection).size(); } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 040e7d1741c..475cccda322 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -288,7 +288,6 @@ message PValue { PArrayAggValue array_agg_value = 64; PCurrentTimestampValue current_timestamp_value = 65; PExplodeValue explode_value = 66; - PArraySortValue array_sort_value = 67; } // Removed types that are no longer supported reserved 38; @@ -1414,17 +1413,14 @@ message PRangeValue { message PExplodeValue { optional PValue collection_value = 1; optional bool preserve_order = 2; + optional bool sorted = 3; + optional bool reverse = 4; } message PArrayDistinctValue { optional PValue child_value = 1; } -message PArraySortValue { - optional PValue child_value = 1; - optional bool reverse = 2; -} - message PParameterObjectValue { optional string parameter_name = 1; optional PType result_type = 2; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 6508addf9c2..1a6707881dc 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -220,8 +220,8 @@ void evaluatingAsAListOfAnEmptyOrNullArrayProducesNothing() { @ParameterizedTest @BooleanSource - void theOrdinalsOfASortedCollectionFollowTheSortedOrder(final boolean reverse) { - final var value = new ExplodeValue(new ArraySortValue(LiteralValue.ofList(List.of(30, 10, 20)), reverse), true); + void sortedExplodeProducesTheElementsSortedAndNumbersThemInThatOrder(final boolean reverse) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of(30, 10, 20)), false, true, reverse); Assertions.assertEquals(reverse ? List.of(List.of(30, 0), List.of(20, 1), List.of(10, 2)) : List.of(List.of(10, 0), List.of(20, 1), List.of(30, 2)), @@ -355,6 +355,49 @@ void equalityAndHashingFollowWhetherTheOrderIsPreserved() { Assertions.assertEquals(new ExplodeValue(otherCollection, true), ordered.withCollectionValue(otherCollection)); } + @Test + void equalityAndHashingFollowTheSortOrder() { + final var collection = LiteralValue.ofList(List.of(1, 2)); + final var value = new ExplodeValue(collection); + final var ascending = value.sorted(false); + final var descending = value.sorted(true); + + Assertions.assertNotEquals(value, ascending); + Assertions.assertNotEquals(ascending, descending); + Assertions.assertEquals(new ExplodeValue(collection, false, true, true), descending); + Assertions.assertNotEquals(ascending.planHash(PlanHashable.CURRENT_FOR_CONTINUATION), + descending.planHash(PlanHashable.CURRENT_FOR_CONTINUATION)); + + // rebasing the collection keeps the sort order + final var otherCollection = LiteralValue.ofList(List.of(3, 4)); + Assertions.assertEquals(new ExplodeValue(otherCollection, false, true, true), + descending.withChildren(List.of(otherCollection))); + } + + @Test + void rejectsSortingAnExplodeThatPreservesItsOrder() { + final var collection = LiteralValue.ofList(List.of(1, 2)); + Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(collection, true, true, false)); + Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(collection, true).sorted(false)); + } + + @Test + void rejectsReversingAnExplodeThatIsNotSorted() { + Assertions.assertThrows(VerifyException.class, + () -> new ExplodeValue(LiteralValue.ofList(List.of(1, 2)), false, false, true)); + } + + @Test + void rejectsSortingElementsThatCannotBeSorted() { + final var records = new LiteralValue<>(new Type.Array(Type.Record.fromFields(List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG), Optional.of("a"))))), List.of()); + Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(records).sorted(false)); + final var bytes = new LiteralValue<>(new Type.Array(Type.primitiveType(Type.TypeCode.BYTES, false)), List.of()); + Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(bytes).sorted(false)); + // an explode of such elements that is not sorted is fine + Assertions.assertFalse(new ExplodeValue(records).isSorted()); + } + @Test void explainsAsAnExplodeOfTheCollection() { final var value = new ExplodeValue(LiteralValue.ofList(List.of(1, 2))); @@ -362,6 +405,14 @@ void explainsAsAnExplodeOfTheCollection() { value.explain().getExplainTokens().render(DefaultExplainFormatter.create(DefaultExplainSymbolMap::new)).toString()); } + @ParameterizedTest + @BooleanSource + void explainsTheSortOrder(final boolean reverse) { + final var value = new ExplodeValue(LiteralValue.ofList(List.of(1, 2))).sorted(reverse); + Assertions.assertEquals(reverse ? "explode([1, 2]) SORTED DESC" : "explode([1, 2]) SORTED", + value.explain().getExplainTokens().render(DefaultExplainFormatter.create(DefaultExplainSymbolMap::new)).toString()); + } + @ParameterizedTest @BooleanSource void derivationsOfATableFunctionKeepTheCollectionCorrelationAndTheStruct(final boolean preserveOrder) { @@ -399,4 +450,16 @@ void protoRoundTrip(final boolean preserveOrder) { Assertions.assertEquals(value.getResultType(), deserialized.getResultType()); Assertions.assertEquals(preserveOrder, ((ExplodeValue)deserialized).preservesOrder()); } + + @ParameterizedTest + @BooleanSource + void protoRoundTripOfASortedExplode(final boolean reverse) { + final var value = new ExplodeValue(AbstractArrayConstructorValue.LightArrayConstructorValue.of( + List.of(LiteralValue.ofScalar(30), LiteralValue.ofScalar(10), LiteralValue.ofScalar(20)))).sorted(reverse); + final var deserialized = Value.fromValueProto(newSerializationContext(), value.toValueProto(newSerializationContext())); + Assertions.assertEquals(value, deserialized); + Assertions.assertTrue(((ExplodeValue)deserialized).isSorted()); + Assertions.assertEquals(reverse, ((ExplodeValue)deserialized).isReverse()); + Assertions.assertEquals(explodeAsList(value, contextFor(value)), explodeAsList((ExplodeValue)deserialized, contextFor(value))); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 7980e6ffdf0..5d0dec91132 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -27,13 +27,11 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; -import com.apple.foundationdb.record.query.plan.cascades.values.ArraySortValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExplodeValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; -import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; import org.junit.jupiter.api.Assertions; @@ -44,7 +42,6 @@ import javax.annotation.Nonnull; import java.util.List; -import java.util.Optional; import java.util.stream.Stream; /** @@ -81,8 +78,8 @@ private static List elementsOf(@Nonnull final InExplodeSource source) { static Stream sources() { return Stream.of( Arguments.of(new InExplodeSource("in", new ExplodeValue(elements())), List.of(30L, 10L, 20L)), - Arguments.of(new InExplodeSource("in", new ExplodeValue(new ArraySortValue(elements(), false))), List.of(10L, 20L, 30L)), - Arguments.of(new InExplodeSource("in", new ExplodeValue(new ArraySortValue(elements(), true))), List.of(30L, 20L, 10L))); + Arguments.of(new InExplodeSource("in", new ExplodeValue(elements()).sorted(false)), List.of(10L, 20L, 30L)), + Arguments.of(new InExplodeSource("in", new ExplodeValue(elements()).sorted(true)), List.of(30L, 20L, 10L))); } @ParameterizedTest @@ -127,33 +124,4 @@ void protoRoundTrip(@Nonnull final InExplodeSource source, @Nonnull final List expected) { - final var collectionValue = new ExplodeValue(new ArraySortValue(elements(), source.isReverse())).getCollectionValue(); - final var valueProto = collectionValue.toValueProto(newSerializationContext()); - Assertions.assertTrue(valueProto.hasArraySortValue()); - - final var deserialized = Value.fromValueProto(newSerializationContext(), valueProto); - Assertions.assertInstanceOf(ArraySortValue.class, deserialized); - Assertions.assertEquals(collectionValue, deserialized); - Assertions.assertEquals(source.isReverse(), ((ArraySortValue)deserialized).isReverse()); - } - - @Test - void arraySortValueRejectsElementsThatCannotBeSorted() { - final var records = new LiteralValue<>(new Type.Array(Type.Record.fromFields(List.of( - Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG), Optional.of("a"))))), List.of()); - Assertions.assertThrows(VerifyException.class, () -> new ArraySortValue(records, false)); - final var bytes = new LiteralValue<>(new Type.Array(Type.primitiveType(Type.TypeCode.BYTES, false)), List.of()); - Assertions.assertThrows(VerifyException.class, () -> new ArraySortValue(bytes, false)); - } - - @Test - void sortedCollectionOfAnExplodeThatPreservesOrderIsRejected() { - final var explodeValue = new ExplodeValue(new ArraySortValue(elements(), false), true); - final var exception = Assertions.assertThrows(VerifyException.class, () -> new InExplodeSource("in", explodeValue)); - Assertions.assertEquals("cannot sort the collection of an explode that preserves its order", exception.getMessage()); - } } From c949542ec0270495994fe9e558be2d4c8dfc6741 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 23:37:40 +0000 Subject: [PATCH 15/16] Check that an explode preserves no order before sorting it --- .../record/query/plan/cascades/values/ExplodeValue.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java index e89eb683b1c..8ff2b7a1400 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValue.java @@ -204,14 +204,16 @@ public ExplodeValue withCollectionValue(@Nonnull final Value newCollectionValue) } /** - * Returns an explode of the same collection that produces its elements sorted. + * Returns an explode of the same collection that produces its elements sorted. Only an explode that does not + * preserve the order of its collection can be sorted. * * @param reverse whether to sort in descending order * @return a sorted explode of the same collection */ @Nonnull public ExplodeValue sorted(final boolean reverse) { - return new ExplodeValue(collectionValue, preserveOrder, true, reverse); + Verify.verify(!preserveOrder, "cannot sort an explode that preserves its order"); + return new ExplodeValue(collectionValue, false, true, reverse); } @Nonnull From a7d806c64e411684436cc2c4ee8b69e0d4c78239 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Thu, 5 Nov 2026 23:39:53 +0000 Subject: [PATCH 16/16] Test the cardinalities and the cursor of an explode, and the type, hash and explain of an in-explode source --- .../cascades/values/ExplodeValueTest.java | 42 +++++++++++++++++++ .../query/plan/plans/InExplodeSourceTest.java | 29 +++++++++++++ 2 files changed, 71 insertions(+) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java index 1a6707881dc..d3055e16a85 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/values/ExplodeValueTest.java @@ -26,7 +26,9 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorVisitor; import com.apple.foundationdb.record.TestRecords4Proto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; @@ -312,6 +314,46 @@ void createsAStructOnlyOnceTheCursorReachesItsElement() { Assertions.assertEquals(1, structsCreated.get()); } + @Test + void theCardinalityOfAnExplodeOfAnArrayConstructorIsItsNumberOfElements() { + final var value = new ExplodeValue(AbstractArrayConstructorValue.LightArrayConstructorValue.of( + List.of(LiteralValue.ofScalar(1), LiteralValue.ofScalar(2), LiteralValue.ofScalar(3)))); + final var cardinalities = value.getCardinalities(); + Assertions.assertEquals(3, cardinalities.getMinCardinality().getCardinality()); + Assertions.assertEquals(3, cardinalities.getMaxCardinality().getCardinality()); + } + + @Test + void theMaximumCardinalityOfAnyOtherExplodeIsUnknown() { + Assertions.assertTrue(new ExplodeValue(reviewsOfRestaurant()).getCardinalities().getMaxCardinality().isUnknown()); + } + + @Test + void theCursorIsALeafOfTheCursorTree() { + final var cursor = new ExplodeValue.Cursor(ForkJoinPool.commonPool(), List.of("a"), + (array, ordinal) -> array.get(ordinal), null); + final var entered = new AtomicInteger(); + final var left = new AtomicInteger(); + final boolean result = cursor.accept(new RecordCursorVisitor() { + @Override + public boolean visitEnter(final RecordCursor visited) { + Assertions.assertSame(cursor, visited); + entered.incrementAndGet(); + return true; + } + + @Override + public boolean visitLeave(final RecordCursor visited) { + Assertions.assertSame(cursor, visited); + left.incrementAndGet(); + return true; + } + }); + Assertions.assertTrue(result); + Assertions.assertEquals(1, entered.get()); + Assertions.assertEquals(1, left.get()); + } + @Test void rejectsACollectionThatIsNotAnArray() { Assertions.assertThrows(VerifyException.class, () -> new ExplodeValue(LiteralValue.ofScalar(1))); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java index 5d0dec91132..89d6ffc4bdf 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/InExplodeSourceTest.java @@ -31,6 +31,8 @@ import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.ParameterObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.explain.DefaultExplainFormatter; +import com.apple.foundationdb.record.query.plan.explain.DefaultExplainSymbolMap; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; @@ -103,6 +105,33 @@ void sizeOfANullCollectionIsZero() { Assertions.assertEquals(2, source.size(EvaluationContext.EMPTY.withBinding("p", List.of(1L, 2L)))); } + @ParameterizedTest + @MethodSource("sources") + void flowsTheStructOfTheExplode(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + final var explodeValue = new ExplodeValue(elements()); + Assertions.assertEquals(explodeValue.getResultType(), source.getResultType()); + } + + @ParameterizedTest + @MethodSource("sources") + void equalSourcesHaveEqualHashCodes(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + final var deserialized = InSource.fromInSourceProto(newSerializationContext(), + source.toInSourceProto(newSerializationContext())); + Assertions.assertEquals(source, deserialized); + Assertions.assertEquals(source.hashCode(), deserialized.hashCode()); + Assertions.assertNotEquals(source, new InExplodeSource("other", new ExplodeValue(elements()))); + } + + @ParameterizedTest + @MethodSource("sources") + void explainsTheCollectionAndTheSortOrder(@Nonnull final InExplodeSource source, @Nonnull final List expected) { + final String explained = source.explain().getExplainTokens() + .render(DefaultExplainFormatter.create(DefaultExplainSymbolMap::new)).toString(); + Assertions.assertTrue(explained.startsWith("IN "), explained); + Assertions.assertEquals(source.isSorted(), explained.contains("SORTED"), explained); + Assertions.assertEquals(source.isReverse(), explained.contains("DESC"), explained); + } + @Test void valuesNeedAnEvaluationContext() { final var source = new InExplodeSource("in", new ExplodeValue(elements()));