From a485f20ed593a0698a7c271ac656eb72fd479f07 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 4 Nov 2026 12:31:30 +0000 Subject: [PATCH 1/4] Offer indexes on synthetic record types to the planner when a query reads all their stored constituents --- .../plan/cascades/MetaDataPlanContext.java | 33 ++++ ...MetaDataPlanContextSyntheticIndexTest.java | 183 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cb3b44cd3d0..eeced654c24 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -31,6 +31,7 @@ import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -102,6 +103,36 @@ private static List readableOf(@Nonnull RecordStoreState recordStoreState } } + /** + * The readable indexes on synthetic record types that a query reading the given record types may be answered from. + * A synthetic record is made of stored records, so an index on a synthetic record type is offered only if the query + * reads every stored record type its constituents are; a constituent that is not stored, such as the element of an + * unnested array, is read through a stored one. + * + * @param metaData the meta-data the synthetic record types are defined in + * @param recordStoreState the state that decides which indexes are readable + * @param queriedRecordTypeNames the names of the stored record types the query reads + * @return the readable indexes on the synthetic record types the query reads all the stored constituents of + */ + @Nonnull + @VisibleForTesting + static List syntheticIndexesOf(@Nonnull final RecordMetaData metaData, + @Nonnull final RecordStoreState recordStoreState, + @Nonnull final Collection queriedRecordTypeNames) { + final var indexes = Lists.newArrayList(); + for (final var syntheticRecordType : metaData.getSyntheticRecordTypes().values()) { + final var storedConstituentNames = + syntheticRecordType.getConstituents().stream() + .map(constituent -> constituent.getRecordType().getName()) + .filter(metaData.getRecordTypes()::containsKey) + .collect(ImmutableSet.toImmutableSet()); + if (!storedConstituentNames.isEmpty() && queriedRecordTypeNames.containsAll(storedConstituentNames)) { + indexes.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } + } + return indexes; + } + @Nonnull public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull RecordMetaData metaData, @@ -144,6 +175,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio } indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes())); + indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); } finally { recordStoreState.endRead(); } @@ -187,6 +219,7 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf for (final var recordType : queriedRecordTypes) { indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes())); } + indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); } finally { recordStoreState.endRead(); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java new file mode 100644 index 00000000000..1f00f280f3f --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java @@ -0,0 +1,183 @@ +/* + * MetaDataPlanContextSyntheticIndexTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.IndexState; +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordMetaDataBuilder; +import com.apple.foundationdb.record.RecordStoreState; +import com.apple.foundationdb.record.TestRecordsNestedMapProto; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder; +import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; +import com.apple.foundationdb.record.query.IndexQueryabilityFilter; +import com.apple.foundationdb.record.query.RecordQuery; +import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static com.apple.foundationdb.record.metadata.Key.Expressions.concat; +import static com.apple.foundationdb.record.metadata.Key.Expressions.field; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests of which indexes on synthetic record types {@link MetaDataPlanContext} offers to the planner. None of these + * tests needs a record store: the readability of an index is all the store state contributes. + */ +class MetaDataPlanContextSyntheticIndexTest { + @Nonnull + private static final String OUTER = "OuterRecord"; + @Nonnull + private static final String OTHER = "OtherRecord"; + @Nonnull + private static final String JOINED_INDEX = "joined_index"; + @Nonnull + private static final String STORED_INDEX = "stored_index"; + @Nonnull + private static final RecordStoreState ALL_READABLE = new RecordStoreState(null, null); + + @Test + void syntheticIndexesOfOffersAJoinedIndexOnlyIfTheQueryReadsEveryJoinedType() { + final RecordMetaData metaData = metaData(addOuterOtherJoined()); + + assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER, OTHER)))) + .containsExactly(JOINED_INDEX); + // A query reading only one of the joined types has no joined records to read, so the index is not offered. + assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER))).isEmpty(); + assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); + } + + @Test + void syntheticIndexesOfOffersASelfJoinedIndexIfTheQueryReadsTheJoinedType() { + final RecordMetaData metaData = metaData(metaDataBuilder -> { + final JoinedRecordTypeBuilder typeBuilder = metaDataBuilder.addJoinedRecordType("SelfJoined"); + typeBuilder.addConstituent("first", OUTER); + typeBuilder.addConstituent("second", OUTER); + typeBuilder.addJoin("first", field("other_id"), "second", field("rec_id")); + metaDataBuilder.addIndex(typeBuilder, new Index(JOINED_INDEX, + concat(field("first").nest("rec_id"), field("second").nest("rec_id")))); + }); + + // Both constituents are the same stored type, so reading that one type reads every constituent. + assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER)))) + .containsExactly(JOINED_INDEX); + assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); + } + + @Test + void syntheticIndexesOfOffersAnUnnestedIndexIfTheQueryReadsTheParentType() { + final RecordMetaData metaData = metaData(metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType("UnnestedMap"); + typeBuilder.addParentConstituent("parent", metaDataBuilder.getRecordType(OUTER)); + typeBuilder.addNestedConstituent("entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + "parent", field("map").nest(field("entry", FanType.FanOut))); + metaDataBuilder.addIndex(typeBuilder, new Index("unnested_index", field("entry").nest("key"))); + }); + + // The nested constituent is not a stored type: it is read through the parent, the only stored constituent. + assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER)))) + .containsExactly("unnested_index"); + assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); + } + + @Test + void syntheticIndexesOfDoesNotOfferAnIndexThatIsNotReadable() { + final RecordMetaData metaData = metaData(addOuterOtherJoined()); + + for (final IndexState state : List.of(IndexState.DISABLED, IndexState.WRITE_ONLY)) { + final RecordStoreState recordStoreState = new RecordStoreState(null, Map.of(JOINED_INDEX, state)); + assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, recordStoreState, Set.of(OUTER, OTHER))) + .as("index in state %s", state) + .isEmpty(); + } + } + + /** + * An index on a synthetic record type is offered to the planner, but cannot be expanded into a match candidate + * yet: the primary key of a synthetic record type is a list, which the expansion does not support. The expansion + * gives up on such an index rather than failing, so offering it leaves planning as it was. + */ + @Test + void planContextForAQueryReadingTheJoinedTypesBuildsWithoutACandidateForTheJoinedIndex() { + final RecordMetaData metaData = metaData(metaDataBuilder -> { + addOuterOtherJoined().accept(metaDataBuilder); + // A query reading both stored types is offered only the indexes on both of them. + metaDataBuilder.addMultiTypeIndex( + List.of(metaDataBuilder.getRecordType(OUTER), metaDataBuilder.getRecordType(OTHER)), + new Index(STORED_INDEX, field("other_id"))); + }); + + final PlanContext forRecordQuery = MetaDataPlanContext.forRecordQuery( + RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), metaData, ALL_READABLE, + IndexMaintainerFactoryRegistryImpl.instance(), + RecordQuery.newBuilder().setRecordTypes(List.of(OUTER, OTHER)).build()); + assertThat(candidateNames(forRecordQuery)).contains(STORED_INDEX).doesNotContain(JOINED_INDEX); + + final Set queriedTypes = Set.of(OUTER, OTHER); + final PlanContext forRootReference = MetaDataPlanContext.forRootReference( + RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), metaData, ALL_READABLE, + IndexMaintainerFactoryRegistryImpl.instance(), + Reference.initialOf(new FullUnorderedScanExpression(queriedTypes, + metaData.getPlannerType(queriedTypes), new AccessHints())), + Optional.empty(), IndexQueryabilityFilter.DEFAULT); + assertThat(candidateNames(forRootReference)).contains(STORED_INDEX).doesNotContain(JOINED_INDEX); + } + + @Nonnull + private static Consumer addOuterOtherJoined() { + return metaDataBuilder -> { + final JoinedRecordTypeBuilder typeBuilder = metaDataBuilder.addJoinedRecordType("OuterOtherJoined"); + typeBuilder.addConstituent("outer", OUTER); + typeBuilder.addConstituent("other", OTHER); + typeBuilder.addJoin("outer", field("other_id"), "other", field("other_id")); + metaDataBuilder.addIndex(typeBuilder, new Index(JOINED_INDEX, + concat(field("outer").nest("rec_id"), field("other").nest("other_value")))); + }; + } + + @Nonnull + private static RecordMetaData metaData(@Nonnull final Consumer hook) { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsNestedMapProto.getDescriptor()); + hook.accept(metaDataBuilder); + return metaDataBuilder.build(); + } + + @Nonnull + private static List indexNames(@Nonnull final List indexes) { + return indexes.stream().map(Index::getName).collect(Collectors.toList()); + } + + @Nonnull + private static Set candidateNames(@Nonnull final PlanContext planContext) { + return planContext.getMatchCandidates().stream().map(MatchCandidate::getName).collect(Collectors.toSet()); + } +} From 424c5530aa12247e9faa712a1a42c3787c786638 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Wed, 4 Nov 2026 13:59:31 +0000 Subject: [PATCH 2/4] nit --- .../record/query/plan/cascades/MetaDataPlanContext.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index eeced654c24..cdb7b1ac39e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -104,10 +104,7 @@ private static List readableOf(@Nonnull RecordStoreState recordStoreState } /** - * The readable indexes on synthetic record types that a query reading the given record types may be answered from. - * A synthetic record is made of stored records, so an index on a synthetic record type is offered only if the query - * reads every stored record type its constituents are; a constituent that is not stored, such as the element of an - * unnested array, is read through a stored one. + * The readable indexes on synthetic record types whose stored constituents are all among the queried record types. * * @param metaData the meta-data the synthetic record types are defined in * @param recordStoreState the state that decides which indexes are readable From 5c4c5dacd293b583bc4809c98b1b441c5dff8219 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 9 Nov 2026 15:54:21 +0000 Subject: [PATCH 3/4] address comments --- .../foundationdb/record/RecordMetaData.java | 21 ++ .../record/metadata/JoinedRecordType.java | 10 + .../record/metadata/SyntheticRecordType.java | 7 + .../record/metadata/UnnestedRecordType.java | 8 + .../plan/cascades/MetaDataPlanContext.java | 49 ++--- .../synthetic/SyntheticRecordPlanner.java | 3 +- .../synthetic/UnnestStoredRecordPlan.java | 5 +- .../plan/synthetic/UnnestedRecordPlanner.java | 2 +- .../metadata/RecordMetaDataBuilderTest.java | 54 ++++++ ...MetaDataPlanContextSyntheticIndexTest.java | 183 ------------------ 10 files changed, 118 insertions(+), 224 deletions(-) delete mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 00aae29c420..9adfd36e6f5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -49,6 +49,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -260,6 +261,26 @@ public SyntheticRecordType getSyntheticRecordType(@Nonnull String name) { return recordType; } + /** + * Get the synthetic record types whose records are all generated from records of the given stored record types. + * That is the case if every stored constituent of the synthetic record type is of one of the given types. + * + * @param recordTypes the stored record types records are read from + * @return the synthetic record types whose records are all generated from records of {@code recordTypes} + */ + @Nonnull + @API(API.Status.EXPERIMENTAL) + public List> getSyntheticRecordTypesFromRecordTypes(@Nonnull Collection recordTypes) { + final List> result = new ArrayList<>(); + for (SyntheticRecordType syntheticRecordType : syntheticRecordTypes.values()) { + final Set storedRecordTypes = syntheticRecordType.getStoredRecordTypes(); + if (!storedRecordTypes.isEmpty() && recordTypes.containsAll(storedRecordTypes)) { + result.add(syntheticRecordType); + } + } + return result; + } + @Nonnull @API(API.Status.EXPERIMENTAL) @SuppressWarnings("squid:S1452") diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java index 89691582288..c7ce62834d7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java @@ -40,6 +40,7 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.tuple.Tuple; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -49,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -152,6 +154,14 @@ protected JoinedRecordType(@Nonnull RecordMetaData metaData, @Nonnull Descriptor this.joins = joins; } + @Nonnull + @Override + public Set getStoredRecordTypes() { + return getConstituents().stream() + .map(JoinConstituent::getRecordType) + .collect(ImmutableSet.toImmutableSet()); + } + @Nonnull public List getJoins() { return joins; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java index dfb409f035b..889129cde03 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java @@ -33,6 +33,7 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; /** @@ -89,6 +90,12 @@ public List getConstituents() { return constituents; } + /** + * {@return the stored record types of the constituents of this type} + */ + @Nonnull + public abstract Set getStoredRecordTypes(); + @Override public boolean isSynthetic() { return true; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java index d3f6a3a2cb8..e29d1353827 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java @@ -55,11 +55,13 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -261,6 +263,12 @@ public NestedConstituent getParentConstituent() { return parentConstituent; } + @Nonnull + @Override + public Set getStoredRecordTypes() { + return Collections.singleton(getParentConstituent().getRecordType()); + } + @Override @Nonnull @API(API.Status.INTERNAL) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cdb7b1ac39e..cdc630501f1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -31,8 +31,8 @@ import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import javax.annotation.Nonnull; @@ -103,33 +103,6 @@ private static List readableOf(@Nonnull RecordStoreState recordStoreState } } - /** - * The readable indexes on synthetic record types whose stored constituents are all among the queried record types. - * - * @param metaData the meta-data the synthetic record types are defined in - * @param recordStoreState the state that decides which indexes are readable - * @param queriedRecordTypeNames the names of the stored record types the query reads - * @return the readable indexes on the synthetic record types the query reads all the stored constituents of - */ - @Nonnull - @VisibleForTesting - static List syntheticIndexesOf(@Nonnull final RecordMetaData metaData, - @Nonnull final RecordStoreState recordStoreState, - @Nonnull final Collection queriedRecordTypeNames) { - final var indexes = Lists.newArrayList(); - for (final var syntheticRecordType : metaData.getSyntheticRecordTypes().values()) { - final var storedConstituentNames = - syntheticRecordType.getConstituents().stream() - .map(constituent -> constituent.getRecordType().getName()) - .filter(metaData.getRecordTypes()::containsKey) - .collect(ImmutableSet.toImmutableSet()); - if (!storedConstituentNames.isEmpty() && queriedRecordTypeNames.containsAll(storedConstituentNames)) { - indexes.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); - } - } - return indexes; - } - @Nonnull public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull RecordMetaData metaData, @@ -142,18 +115,17 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio final var isSortReverse = query.isSortReverse(); List indexList = new ArrayList<>(); - final Set queriedRecordTypeNames; + final Collection queriedRecordTypes; final KeyExpression commonPrimaryKey; recordStoreState.beginRead(); try { if (queriedRecordTypeNamesOptional.isEmpty()) { // ALL_TYPES commonPrimaryKey = commonPrimaryKey(metaData.getRecordTypes().values()); - queriedRecordTypeNames = metaData.getRecordTypes().keySet(); + queriedRecordTypes = metaData.getRecordTypes().values(); } else { - queriedRecordTypeNames = ImmutableSet.copyOf(queriedRecordTypeNamesOptional.get()); - final List queriedRecordTypes = queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList()); + queriedRecordTypes = queriedRecordTypeNamesOptional.get().stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet()); if (queriedRecordTypes.size() == 1) { - final RecordType recordType = queriedRecordTypes.get(0); + final RecordType recordType = Iterables.getOnlyElement(queriedRecordTypes); indexList.addAll(readableOf(recordStoreState, recordType.getIndexes())); indexList.addAll(readableOf(recordStoreState, recordType.getMultiTypeIndexes())); commonPrimaryKey = recordType.getPrimaryKey(); @@ -172,7 +144,9 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio } indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes())); - indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); + for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) { + indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } } finally { recordStoreState.endRead(); } @@ -189,6 +163,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, isSortReverse)); } + final Set queriedRecordTypeNames = queriedRecordTypes.stream().map(RecordType::getName).collect(ImmutableSet.toImmutableSet()); MatchCandidateExpansion.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse) .ifPresent(matchCandidatesBuilder::add); @@ -209,14 +184,16 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf } final var queriedRecordTypes = - queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList()); + queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet()); final var indexList = Lists.newArrayList(); recordStoreState.beginRead(); try { for (final var recordType : queriedRecordTypes) { indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes())); } - indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); + for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) { + indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } } finally { recordStoreState.endRead(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java index 59a12bbcb01..19edc8235b2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java @@ -164,8 +164,7 @@ public SyntheticRecordFromStoredRecordPlan forType(@Nonnull UnnestedRecordType u if (unnestedRecordType.getRecordMetaData() != recordMetaData) { throw mismatchedMetaData(); } - UnnestedRecordType.NestedConstituent parentConstituent = unnestedRecordType.getParentConstituent(); - return new UnnestStoredRecordPlan(unnestedRecordType, parentConstituent.getRecordType()); + return new UnnestStoredRecordPlan(unnestedRecordType); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java index 0bfcbf1d1b8..81f897450b1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBSyntheticRecord; import com.apple.foundationdb.tuple.Tuple; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.protobuf.Message; @@ -66,9 +67,9 @@ class UnnestStoredRecordPlan implements SyntheticRecordFromStoredRecordPlan { @Nonnull private final RecordType storedRecordType; - UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType, @Nonnull RecordType storedRecordType) { + UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType) { this.recordType = recordType; - this.storedRecordType = storedRecordType; + this.storedRecordType = Iterables.getOnlyElement(recordType.getStoredRecordTypes()); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java index 539755eb000..e7ac0eb7c91 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java @@ -45,6 +45,6 @@ public SyntheticRecordFromStoredRecordPlan plan(UnnestedRecordType.NestedConstit .addLogInfo(LogMessageKeys.EXPECTED, recordType.getParentConstituent().getName()) .addLogInfo(LogMessageKeys.ACTUAL, constituent.getName()); } - return new UnnestStoredRecordPlan(recordType, constituent.getRecordType()); + return new UnnestStoredRecordPlan(recordType); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java index 2a90f021018..c93b9f5e613 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.TestRecordsImportFlatProto; import com.apple.foundationdb.record.TestRecordsImportProto; import com.apple.foundationdb.record.TestRecordsMarkedUnmarkedProto; +import com.apple.foundationdb.record.TestRecordsNestedMapProto; import com.apple.foundationdb.record.TestRecordsNoPrimaryKeyProto; import com.apple.foundationdb.record.TestRecordsUnionMissingRecordProto; import com.apple.foundationdb.record.TestRecordsUnionWithImportedNestedProto; @@ -59,8 +60,10 @@ import org.junit.jupiter.params.ParameterizedTest; import javax.annotation.Nonnull; +import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -802,6 +805,57 @@ public void counterBasedSubspaceKeysProtoSettings() { assertEquals("Error converting from protobuf", e.getMessage()); } + @Test + void storedRecordTypesOfSyntheticRecordTypes() { + final RecordMetaData metaData = syntheticRecordTypesMetaData(); + + final RecordType outer = metaData.getRecordType("OuterRecord"); + final RecordType other = metaData.getRecordType("OtherRecord"); + assertEquals(Set.of(outer, other), metaData.getSyntheticRecordType("OuterOtherJoined").getStoredRecordTypes()); + // A type joined to itself is a single stored type. + assertEquals(Set.of(outer), metaData.getSyntheticRecordType("SelfJoined").getStoredRecordTypes()); + // The nested constituent is a message within the parent, not a stored type. + assertEquals(Set.of(outer), metaData.getSyntheticRecordType("UnnestedMap").getStoredRecordTypes()); + } + + @Test + void syntheticRecordTypesFromStoredRecordTypes() { + final RecordMetaData metaData = syntheticRecordTypesMetaData(); + + final RecordType outer = metaData.getRecordType("OuterRecord"); + final RecordType other = metaData.getRecordType("OtherRecord"); + assertEquals(Set.of("OuterOtherJoined", "SelfJoined", "UnnestedMap"), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer, other)))); + assertEquals(Set.of("SelfJoined", "UnnestedMap"), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer)))); + assertEquals(Set.of(), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(other)))); + } + + @Nonnull + private static RecordMetaData syntheticRecordTypesMetaData() { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsNestedMapProto.getDescriptor()); + final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("OuterOtherJoined"); + joined.addConstituent("outer", "OuterRecord"); + joined.addConstituent("other", "OtherRecord"); + joined.addJoin("outer", field("other_id"), "other", field("other_id")); + final JoinedRecordTypeBuilder selfJoined = metaDataBuilder.addJoinedRecordType("SelfJoined"); + selfJoined.addConstituent("first", "OuterRecord"); + selfJoined.addConstituent("second", "OuterRecord"); + selfJoined.addJoin("first", field("other_id"), "second", field("rec_id")); + final UnnestedRecordTypeBuilder unnested = metaDataBuilder.addUnnestedRecordType("UnnestedMap"); + unnested.addParentConstituent("parent", metaDataBuilder.getRecordType("OuterRecord")); + unnested.addNestedConstituent("entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + "parent", field("map").nest(field("entry", KeyExpression.FanType.FanOut))); + return metaDataBuilder.build(); + } + + @Nonnull + private static Set syntheticRecordTypeNames(@Nonnull final Collection> syntheticRecordTypes) { + return syntheticRecordTypes.stream().map(RecordType::getName).collect(Collectors.toSet()); + } + @Test void canSerializeAndDeserializeSyntheticRecordTypes() { RecordMetaDataBuilder rmd = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java deleted file mode 100644 index 1f00f280f3f..00000000000 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContextSyntheticIndexTest.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * MetaDataPlanContextSyntheticIndexTest.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.query.plan.cascades; - -import com.apple.foundationdb.record.IndexState; -import com.apple.foundationdb.record.RecordMetaData; -import com.apple.foundationdb.record.RecordMetaDataBuilder; -import com.apple.foundationdb.record.RecordStoreState; -import com.apple.foundationdb.record.TestRecordsNestedMapProto; -import com.apple.foundationdb.record.metadata.Index; -import com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder; -import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; -import com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType; -import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; -import com.apple.foundationdb.record.query.IndexQueryabilityFilter; -import com.apple.foundationdb.record.query.RecordQuery; -import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; -import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; -import org.junit.jupiter.api.Test; - -import javax.annotation.Nonnull; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static com.apple.foundationdb.record.metadata.Key.Expressions.concat; -import static com.apple.foundationdb.record.metadata.Key.Expressions.field; -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests of which indexes on synthetic record types {@link MetaDataPlanContext} offers to the planner. None of these - * tests needs a record store: the readability of an index is all the store state contributes. - */ -class MetaDataPlanContextSyntheticIndexTest { - @Nonnull - private static final String OUTER = "OuterRecord"; - @Nonnull - private static final String OTHER = "OtherRecord"; - @Nonnull - private static final String JOINED_INDEX = "joined_index"; - @Nonnull - private static final String STORED_INDEX = "stored_index"; - @Nonnull - private static final RecordStoreState ALL_READABLE = new RecordStoreState(null, null); - - @Test - void syntheticIndexesOfOffersAJoinedIndexOnlyIfTheQueryReadsEveryJoinedType() { - final RecordMetaData metaData = metaData(addOuterOtherJoined()); - - assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER, OTHER)))) - .containsExactly(JOINED_INDEX); - // A query reading only one of the joined types has no joined records to read, so the index is not offered. - assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER))).isEmpty(); - assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); - } - - @Test - void syntheticIndexesOfOffersASelfJoinedIndexIfTheQueryReadsTheJoinedType() { - final RecordMetaData metaData = metaData(metaDataBuilder -> { - final JoinedRecordTypeBuilder typeBuilder = metaDataBuilder.addJoinedRecordType("SelfJoined"); - typeBuilder.addConstituent("first", OUTER); - typeBuilder.addConstituent("second", OUTER); - typeBuilder.addJoin("first", field("other_id"), "second", field("rec_id")); - metaDataBuilder.addIndex(typeBuilder, new Index(JOINED_INDEX, - concat(field("first").nest("rec_id"), field("second").nest("rec_id")))); - }); - - // Both constituents are the same stored type, so reading that one type reads every constituent. - assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER)))) - .containsExactly(JOINED_INDEX); - assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); - } - - @Test - void syntheticIndexesOfOffersAnUnnestedIndexIfTheQueryReadsTheParentType() { - final RecordMetaData metaData = metaData(metaDataBuilder -> { - final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType("UnnestedMap"); - typeBuilder.addParentConstituent("parent", metaDataBuilder.getRecordType(OUTER)); - typeBuilder.addNestedConstituent("entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), - "parent", field("map").nest(field("entry", FanType.FanOut))); - metaDataBuilder.addIndex(typeBuilder, new Index("unnested_index", field("entry").nest("key"))); - }); - - // The nested constituent is not a stored type: it is read through the parent, the only stored constituent. - assertThat(indexNames(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OUTER)))) - .containsExactly("unnested_index"); - assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, ALL_READABLE, Set.of(OTHER))).isEmpty(); - } - - @Test - void syntheticIndexesOfDoesNotOfferAnIndexThatIsNotReadable() { - final RecordMetaData metaData = metaData(addOuterOtherJoined()); - - for (final IndexState state : List.of(IndexState.DISABLED, IndexState.WRITE_ONLY)) { - final RecordStoreState recordStoreState = new RecordStoreState(null, Map.of(JOINED_INDEX, state)); - assertThat(MetaDataPlanContext.syntheticIndexesOf(metaData, recordStoreState, Set.of(OUTER, OTHER))) - .as("index in state %s", state) - .isEmpty(); - } - } - - /** - * An index on a synthetic record type is offered to the planner, but cannot be expanded into a match candidate - * yet: the primary key of a synthetic record type is a list, which the expansion does not support. The expansion - * gives up on such an index rather than failing, so offering it leaves planning as it was. - */ - @Test - void planContextForAQueryReadingTheJoinedTypesBuildsWithoutACandidateForTheJoinedIndex() { - final RecordMetaData metaData = metaData(metaDataBuilder -> { - addOuterOtherJoined().accept(metaDataBuilder); - // A query reading both stored types is offered only the indexes on both of them. - metaDataBuilder.addMultiTypeIndex( - List.of(metaDataBuilder.getRecordType(OUTER), metaDataBuilder.getRecordType(OTHER)), - new Index(STORED_INDEX, field("other_id"))); - }); - - final PlanContext forRecordQuery = MetaDataPlanContext.forRecordQuery( - RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), metaData, ALL_READABLE, - IndexMaintainerFactoryRegistryImpl.instance(), - RecordQuery.newBuilder().setRecordTypes(List.of(OUTER, OTHER)).build()); - assertThat(candidateNames(forRecordQuery)).contains(STORED_INDEX).doesNotContain(JOINED_INDEX); - - final Set queriedTypes = Set.of(OUTER, OTHER); - final PlanContext forRootReference = MetaDataPlanContext.forRootReference( - RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), metaData, ALL_READABLE, - IndexMaintainerFactoryRegistryImpl.instance(), - Reference.initialOf(new FullUnorderedScanExpression(queriedTypes, - metaData.getPlannerType(queriedTypes), new AccessHints())), - Optional.empty(), IndexQueryabilityFilter.DEFAULT); - assertThat(candidateNames(forRootReference)).contains(STORED_INDEX).doesNotContain(JOINED_INDEX); - } - - @Nonnull - private static Consumer addOuterOtherJoined() { - return metaDataBuilder -> { - final JoinedRecordTypeBuilder typeBuilder = metaDataBuilder.addJoinedRecordType("OuterOtherJoined"); - typeBuilder.addConstituent("outer", OUTER); - typeBuilder.addConstituent("other", OTHER); - typeBuilder.addJoin("outer", field("other_id"), "other", field("other_id")); - metaDataBuilder.addIndex(typeBuilder, new Index(JOINED_INDEX, - concat(field("outer").nest("rec_id"), field("other").nest("other_value")))); - }; - } - - @Nonnull - private static RecordMetaData metaData(@Nonnull final Consumer hook) { - final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() - .setRecords(TestRecordsNestedMapProto.getDescriptor()); - hook.accept(metaDataBuilder); - return metaDataBuilder.build(); - } - - @Nonnull - private static List indexNames(@Nonnull final List indexes) { - return indexes.stream().map(Index::getName).collect(Collectors.toList()); - } - - @Nonnull - private static Set candidateNames(@Nonnull final PlanContext planContext) { - return planContext.getMatchCandidates().stream().map(MatchCandidate::getName).collect(Collectors.toSet()); - } -} From d4554872c0778fc5402970c215cbfdacf6f88424 Mon Sep 17 00:00:00 2001 From: Pranjal Gupta Date: Mon, 9 Nov 2026 18:50:41 +0000 Subject: [PATCH 4/4] address comments --- .../java/com/apple/foundationdb/record/RecordMetaData.java | 3 ++- .../record/query/plan/cascades/MetaDataPlanContext.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 9adfd36e6f5..1d8b4ab728d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -274,7 +274,8 @@ public List> getSyntheticRecordTypesFromRecordTypes(@Nonn final List> result = new ArrayList<>(); for (SyntheticRecordType syntheticRecordType : syntheticRecordTypes.values()) { final Set storedRecordTypes = syntheticRecordType.getStoredRecordTypes(); - if (!storedRecordTypes.isEmpty() && recordTypes.containsAll(storedRecordTypes)) { + Verify.verify(!storedRecordTypes.isEmpty()); + if (recordTypes.containsAll(storedRecordTypes)) { result.add(syntheticRecordType); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cdc630501f1..49afd69f34b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -121,7 +121,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio try { if (queriedRecordTypeNamesOptional.isEmpty()) { // ALL_TYPES commonPrimaryKey = commonPrimaryKey(metaData.getRecordTypes().values()); - queriedRecordTypes = metaData.getRecordTypes().values(); + queriedRecordTypes = ImmutableSet.copyOf(metaData.getRecordTypes().values()); } else { queriedRecordTypes = queriedRecordTypeNamesOptional.get().stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet()); if (queriedRecordTypes.size() == 1) {