Offer indexes on synthetic record types to the planner when a query reads all their stored constituents

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -48,7 +48,8 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
@@ -260,6 +261,27 @@
         return recordType;
     }
 
+    /**
+     * Get the synthetic record types whose records are all generated from records of the given stored record types.
+     * That is the case if every stored constituent of the synthetic record type is of one of the given types.
+     *
+     * @param recordTypes the stored record types records are read from
+     * @return the synthetic record types whose records are all generated from records of {@code recordTypes}
+     */
+    @Nonnull
+    @API(API.Status.EXPERIMENTAL)
+    public List<SyntheticRecordType<?>> getSyntheticRecordTypesFromRecordTypes(@Nonnull Collection<RecordType> recordTypes) {
+        final List<SyntheticRecordType<?>> result = new ArrayList<>();
+        for (SyntheticRecordType<?> syntheticRecordType : syntheticRecordTypes.values()) {
+            final Set<RecordType> storedRecordTypes = syntheticRecordType.getStoredRecordTypes();
+            Verify.verify(!storedRecordTypes.isEmpty());
+            if (recordTypes.containsAll(storedRecordTypes)) {
+                result.add(syntheticRecordType);
+            }
+        }
+        return result;
+    }
+
     @Nonnull
     @API(API.Status.EXPERIMENTAL)
     @SuppressWarnings("squid:S1452")

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java

@@ -39,7 +39,8 @@
 import com.apple.foundationdb.record.query.plan.cascades.ScalarTranslationVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.tuple.Tuple;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.protobuf.Descriptors;
 import com.google.protobuf.Message;
@@ -49,6 +50,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -152,6 +154,14 @@
         this.joins = joins;
     }
 
+    @Nonnull
+    @Override
+    public Set<RecordType> getStoredRecordTypes() {
+        return getConstituents().stream()
+                .map(JoinConstituent::getRecordType)
+                .collect(ImmutableSet.toImmutableSet());
+    }
+
     @Nonnull
     public List<Join> getJoins() {
         return joins;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java

@@ -32,7 +32,8 @@
 import com.google.protobuf.Descriptors;
 
 import javax.annotation.Nonnull;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -89,6 +90,12 @@
         return constituents;
     }
 
+    /**
+     * {@return the stored record types of the constituents of this type}
+     */
+    @Nonnull
+    public abstract Set<RecordType> getStoredRecordTypes();
+
     @Override
     public boolean isSynthetic() {
         return true;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java

@@ -54,12 +54,14 @@
 import com.google.protobuf.Message;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 
 
@@ -261,6 +263,12 @@
         return parentConstituent;
     }
 
+    @Nonnull
+    @Override
+    public Set<RecordType> getStoredRecordTypes() {
+        return Collections.singleton(getParentConstituent().getRecordType());
+    }
+
     @Override
     @Nonnull
     @API(API.Status.INTERNAL)

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java

@@ -31,7 +31,8 @@
 import com.apple.foundationdb.record.query.IndexQueryabilityFilter;
 import com.apple.foundationdb.record.query.RecordQuery;
 import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 
 import javax.annotation.Nonnull;
@@ -108,62 +109,65 @@
                                              @Nonnull RecordStoreState recordStoreState,
                                              @Nonnull IndexMatchCandidateRegistry matchCandidateRegistry,
                                              @Nonnull RecordQuery query) {
         final Optional<Collection<String>> queriedRecordTypeNamesOptional = query.getRecordTypes().isEmpty() ? Optional.empty() : Optional.of(query.getRecordTypes());
         final Optional<Collection<String>> allowedIndexesOptional = query.hasAllowedIndexes() ? Optional.of(Objects.requireNonNull(query.getAllowedIndexes())) : Optional.empty();
         final var indexQueryabilityFilter = query.getIndexQueryabilityFilter();
         final var isSortReverse = query.isSortReverse();
 
         List<Index> indexList = new ArrayList<>();
-        final Set<String> queriedRecordTypeNames;
+        final Collection<RecordType> queriedRecordTypes;
         final KeyExpression commonPrimaryKey;
         recordStoreState.beginRead();
         try {
             if (queriedRecordTypeNamesOptional.isEmpty()) { // ALL_TYPES
                 commonPrimaryKey = commonPrimaryKey(metaData.getRecordTypes().values());
-                queriedRecordTypeNames = metaData.getRecordTypes().keySet();
+                queriedRecordTypes = ImmutableSet.copyOf(metaData.getRecordTypes().values());
             } else {
-                queriedRecordTypeNames = ImmutableSet.copyOf(queriedRecordTypeNamesOptional.get());
-                final List<RecordType> queriedRecordTypes = queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList());
+                queriedRecordTypes = queriedRecordTypeNamesOptional.get().stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet());
                 if (queriedRecordTypes.size() == 1) {
-                    final RecordType recordType = queriedRecordTypes.get(0);
+                    final RecordType recordType = Iterables.getOnlyElement(queriedRecordTypes);
                     indexList.addAll(readableOf(recordStoreState, recordType.getIndexes()));
                     indexList.addAll(readableOf(recordStoreState, recordType.getMultiTypeIndexes()));
                     commonPrimaryKey = recordType.getPrimaryKey();
                 } else {
                     boolean first = true;
                     for (RecordType recordType : queriedRecordTypes) {
                         if (first) {
                             indexList.addAll(readableOf(recordStoreState, recordType.getMultiTypeIndexes()));
                             first = false;
                         } else {
                             indexList.retainAll(readableOf(recordStoreState, recordType.getMultiTypeIndexes()));
                         }
                     }
                     commonPrimaryKey = commonPrimaryKey(queriedRecordTypes);
                 }
             }
 
             indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes()));
+            for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) {
+                indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes()));
+            }
         } finally {
             recordStoreState.endRead();
         }
 
         if (allowedIndexesOptional.isPresent()) {
             final Collection<String> allowedIndexes = allowedIndexesOptional.get();
             indexList.removeIf(index -> !allowedIndexes.contains(index.getName()));
         } else {
             indexList.removeIf(index -> !indexQueryabilityFilter.isQueryable(index));
         }
 
         final ImmutableSet.Builder<MatchCandidate> matchCandidatesBuilder = ImmutableSet.builder();
         for (Index index : indexList) {
             matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, isSortReverse));
         }
 
+        final Set<String> queriedRecordTypeNames = queriedRecordTypes.stream().map(RecordType::getName).collect(ImmutableSet.toImmutableSet());
         MatchCandidateExpansion.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse)
                 .ifPresent(matchCandidatesBuilder::add);
 
         return new MetaDataPlanContext(plannerConfiguration, matchCandidatesBuilder.build());
     }
 
     public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConfiguration plannerConfiguration,
@@ -180,13 +184,16 @@
         }
 
         final var queriedRecordTypes =
-                queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList());
+                queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet());
         final var indexList = Lists.<Index>newArrayList();
         recordStoreState.beginRead();
         try {
             for (final var recordType : queriedRecordTypes) {
                 indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes()));
             }
+            for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) {
+                indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes()));
+            }
         } finally {
             recordStoreState.endRead();
         }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java

@@ -163,9 +163,8 @@
     public SyntheticRecordFromStoredRecordPlan forType(@Nonnull UnnestedRecordType unnestedRecordType) {
         if (unnestedRecordType.getRecordMetaData() != recordMetaData) {
             throw mismatchedMetaData();
         }
-        UnnestedRecordType.NestedConstituent parentConstituent = unnestedRecordType.getParentConstituent();
-        return new UnnestStoredRecordPlan(unnestedRecordType, parentConstituent.getRecordType());
+        return new UnnestStoredRecordPlan(unnestedRecordType);
     }
 
     /**

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java

@@ -33,7 +33,8 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord;
 import com.apple.foundationdb.record.provider.foundationdb.FDBSyntheticRecord;
 import com.apple.foundationdb.tuple.Tuple;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.protobuf.Message;
 
@@ -66,9 +67,9 @@
     @Nonnull
     private final RecordType storedRecordType;
 
-    UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType, @Nonnull RecordType storedRecordType) {
+    UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType) {
         this.recordType = recordType;
-        this.storedRecordType = storedRecordType;
+        this.storedRecordType = Iterables.getOnlyElement(recordType.getStoredRecordTypes());
     }
 
     @Override

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java

@@ -44,7 +44,7 @@
             throw new RecordCoreArgumentException("Can only create synthetic records for parent constituent")
                     .addLogInfo(LogMessageKeys.EXPECTED, recordType.getParentConstituent().getName())
                     .addLogInfo(LogMessageKeys.ACTUAL, constituent.getName());
         }
-        return new UnnestStoredRecordPlan(recordType, constituent.getRecordType());
+        return new UnnestStoredRecordPlan(recordType);
     }
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java

@@ -35,6 +35,7 @@
 import com.apple.foundationdb.record.TestRecordsImportFlatProto;
 import com.apple.foundationdb.record.TestRecordsImportProto;
 import com.apple.foundationdb.record.TestRecordsMarkedUnmarkedProto;
+import com.apple.foundationdb.record.TestRecordsNestedMapProto;
 import com.apple.foundationdb.record.TestRecordsNoPrimaryKeyProto;
 import com.apple.foundationdb.record.TestRecordsUnionMissingRecordProto;
 import com.apple.foundationdb.record.TestRecordsUnionWithImportedNestedProto;
@@ -59,8 +60,10 @@
 import org.junit.jupiter.params.ParameterizedTest;
 
 import javax.annotation.Nonnull;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
 
@@ -802,6 +805,57 @@ public void counterBasedSubspaceKeysProtoSettings() {
         assertEquals("Error converting from protobuf", e.getMessage());
     }
 
+    @Test
+    void storedRecordTypesOfSyntheticRecordTypes() {
+        final RecordMetaData metaData = syntheticRecordTypesMetaData();
+
+        final RecordType outer = metaData.getRecordType("OuterRecord");
+        final RecordType other = metaData.getRecordType("OtherRecord");
+        assertEquals(Set.of(outer, other), metaData.getSyntheticRecordType("OuterOtherJoined").getStoredRecordTypes());
+        // A type joined to itself is a single stored type.
+        assertEquals(Set.of(outer), metaData.getSyntheticRecordType("SelfJoined").getStoredRecordTypes());
+        // The nested constituent is a message within the parent, not a stored type.
+        assertEquals(Set.of(outer), metaData.getSyntheticRecordType("UnnestedMap").getStoredRecordTypes());
+    }
+
+    @Test
+    void syntheticRecordTypesFromStoredRecordTypes() {
+        final RecordMetaData metaData = syntheticRecordTypesMetaData();
+
+        final RecordType outer = metaData.getRecordType("OuterRecord");
+        final RecordType other = metaData.getRecordType("OtherRecord");
+        assertEquals(Set.of("OuterOtherJoined", "SelfJoined", "UnnestedMap"),
+                syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer, other))));
+        assertEquals(Set.of("SelfJoined", "UnnestedMap"),
+                syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer))));
+        assertEquals(Set.of(),
+                syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(other))));
+    }
+
+    @Nonnull
+    private static RecordMetaData syntheticRecordTypesMetaData() {
+        final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder()
+                .setRecords(TestRecordsNestedMapProto.getDescriptor());
+        final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("OuterOtherJoined");
+        joined.addConstituent("outer", "OuterRecord");
+        joined.addConstituent("other", "OtherRecord");
+        joined.addJoin("outer", field("other_id"), "other", field("other_id"));
+        final JoinedRecordTypeBuilder selfJoined = metaDataBuilder.addJoinedRecordType("SelfJoined");
+        selfJoined.addConstituent("first", "OuterRecord");
+        selfJoined.addConstituent("second", "OuterRecord");
+        selfJoined.addJoin("first", field("other_id"), "second", field("rec_id"));
+        final UnnestedRecordTypeBuilder unnested = metaDataBuilder.addUnnestedRecordType("UnnestedMap");
+        unnested.addParentConstituent("parent", metaDataBuilder.getRecordType("OuterRecord"));
+        unnested.addNestedConstituent("entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(),
+                "parent", field("map").nest(field("entry", KeyExpression.FanType.FanOut)));
+        return metaDataBuilder.build();
+    }
+
+    @Nonnull
+    private static Set<String> syntheticRecordTypeNames(@Nonnull final Collection<SyntheticRecordType<?>> syntheticRecordTypes) {
+        return syntheticRecordTypes.stream().map(RecordType::getName).collect(Collectors.toSet());
+    }
+
     @Test
     void canSerializeAndDeserializeSyntheticRecordTypes() {
         RecordMetaDataBuilder rmd = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());