diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 00aae29c42..1d8b4ab728 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -49,6 +49,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -260,6 +261,27 @@ public SyntheticRecordType getSyntheticRecordType(@Nonnull String name) { return recordType; } + /** + * Get the synthetic record types whose records are all generated from records of the given stored record types. + * That is the case if every stored constituent of the synthetic record type is of one of the given types. + * + * @param recordTypes the stored record types records are read from + * @return the synthetic record types whose records are all generated from records of {@code recordTypes} + */ + @Nonnull + @API(API.Status.EXPERIMENTAL) + public List> getSyntheticRecordTypesFromRecordTypes(@Nonnull Collection recordTypes) { + final List> result = new ArrayList<>(); + for (SyntheticRecordType syntheticRecordType : syntheticRecordTypes.values()) { + final Set storedRecordTypes = syntheticRecordType.getStoredRecordTypes(); + Verify.verify(!storedRecordTypes.isEmpty()); + if (recordTypes.containsAll(storedRecordTypes)) { + result.add(syntheticRecordType); + } + } + return result; + } + @Nonnull @API(API.Status.EXPERIMENTAL) @SuppressWarnings("squid:S1452") diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java index 8969158228..c7ce62834d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/JoinedRecordType.java @@ -40,6 +40,7 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.tuple.Tuple; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -49,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -152,6 +154,14 @@ protected JoinedRecordType(@Nonnull RecordMetaData metaData, @Nonnull Descriptor this.joins = joins; } + @Nonnull + @Override + public Set getStoredRecordTypes() { + return getConstituents().stream() + .map(JoinConstituent::getRecordType) + .collect(ImmutableSet.toImmutableSet()); + } + @Nonnull public List getJoins() { return joins; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java index dfb409f035..889129cde0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java @@ -33,6 +33,7 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; /** @@ -89,6 +90,12 @@ public List getConstituents() { return constituents; } + /** + * {@return the stored record types of the constituents of this type} + */ + @Nonnull + public abstract Set getStoredRecordTypes(); + @Override public boolean isSynthetic() { return true; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java index d3f6a3a2cb..e29d135382 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java @@ -55,11 +55,13 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -261,6 +263,12 @@ public NestedConstituent getParentConstituent() { return parentConstituent; } + @Nonnull + @Override + public Set getStoredRecordTypes() { + return Collections.singleton(getParentConstituent().getRecordType()); + } + @Override @Nonnull @API(API.Status.INTERNAL) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cb3b44cd3d..49afd69f34 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -32,6 +32,7 @@ import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import javax.annotation.Nonnull; @@ -114,18 +115,17 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio final var isSortReverse = query.isSortReverse(); List indexList = new ArrayList<>(); - final Set queriedRecordTypeNames; + final Collection queriedRecordTypes; final KeyExpression commonPrimaryKey; recordStoreState.beginRead(); try { if (queriedRecordTypeNamesOptional.isEmpty()) { // ALL_TYPES commonPrimaryKey = commonPrimaryKey(metaData.getRecordTypes().values()); - queriedRecordTypeNames = metaData.getRecordTypes().keySet(); + queriedRecordTypes = ImmutableSet.copyOf(metaData.getRecordTypes().values()); } else { - queriedRecordTypeNames = ImmutableSet.copyOf(queriedRecordTypeNamesOptional.get()); - final List queriedRecordTypes = queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList()); + queriedRecordTypes = queriedRecordTypeNamesOptional.get().stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet()); if (queriedRecordTypes.size() == 1) { - final RecordType recordType = queriedRecordTypes.get(0); + final RecordType recordType = Iterables.getOnlyElement(queriedRecordTypes); indexList.addAll(readableOf(recordStoreState, recordType.getIndexes())); indexList.addAll(readableOf(recordStoreState, recordType.getMultiTypeIndexes())); commonPrimaryKey = recordType.getPrimaryKey(); @@ -144,6 +144,9 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio } indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes())); + for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) { + indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } } finally { recordStoreState.endRead(); } @@ -160,6 +163,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio matchCandidatesBuilder.addAll(matchCandidateRegistry.createMatchCandidates(metaData, index, isSortReverse)); } + final Set queriedRecordTypeNames = queriedRecordTypes.stream().map(RecordType::getName).collect(ImmutableSet.toImmutableSet()); MatchCandidateExpansion.fromPrimaryDefinition(metaData, queriedRecordTypeNames, commonPrimaryKey, isSortReverse) .ifPresent(matchCandidatesBuilder::add); @@ -180,13 +184,16 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf } final var queriedRecordTypes = - queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList()); + queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(ImmutableSet.toImmutableSet()); final var indexList = Lists.newArrayList(); recordStoreState.beginRead(); try { for (final var recordType : queriedRecordTypes) { indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes())); } + for (final var syntheticRecordType : metaData.getSyntheticRecordTypesFromRecordTypes(queriedRecordTypes)) { + indexList.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } } finally { recordStoreState.endRead(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java index 59a12bbcb0..19edc8235b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/SyntheticRecordPlanner.java @@ -164,8 +164,7 @@ public SyntheticRecordFromStoredRecordPlan forType(@Nonnull UnnestedRecordType u if (unnestedRecordType.getRecordMetaData() != recordMetaData) { throw mismatchedMetaData(); } - UnnestedRecordType.NestedConstituent parentConstituent = unnestedRecordType.getParentConstituent(); - return new UnnestStoredRecordPlan(unnestedRecordType, parentConstituent.getRecordType()); + return new UnnestStoredRecordPlan(unnestedRecordType); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java index 0bfcbf1d1b..81f897450b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestStoredRecordPlan.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBSyntheticRecord; import com.apple.foundationdb.tuple.Tuple; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.protobuf.Message; @@ -66,9 +67,9 @@ class UnnestStoredRecordPlan implements SyntheticRecordFromStoredRecordPlan { @Nonnull private final RecordType storedRecordType; - UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType, @Nonnull RecordType storedRecordType) { + UnnestStoredRecordPlan(@Nonnull UnnestedRecordType recordType) { this.recordType = recordType; - this.storedRecordType = storedRecordType; + this.storedRecordType = Iterables.getOnlyElement(recordType.getStoredRecordTypes()); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java index 539755eb00..e7ac0eb7c9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/synthetic/UnnestedRecordPlanner.java @@ -45,6 +45,6 @@ public SyntheticRecordFromStoredRecordPlan plan(UnnestedRecordType.NestedConstit .addLogInfo(LogMessageKeys.EXPECTED, recordType.getParentConstituent().getName()) .addLogInfo(LogMessageKeys.ACTUAL, constituent.getName()); } - return new UnnestStoredRecordPlan(recordType, constituent.getRecordType()); + return new UnnestStoredRecordPlan(recordType); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java index 2a90f02101..c93b9f5e61 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/RecordMetaDataBuilderTest.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.TestRecordsImportFlatProto; import com.apple.foundationdb.record.TestRecordsImportProto; import com.apple.foundationdb.record.TestRecordsMarkedUnmarkedProto; +import com.apple.foundationdb.record.TestRecordsNestedMapProto; import com.apple.foundationdb.record.TestRecordsNoPrimaryKeyProto; import com.apple.foundationdb.record.TestRecordsUnionMissingRecordProto; import com.apple.foundationdb.record.TestRecordsUnionWithImportedNestedProto; @@ -59,8 +60,10 @@ import org.junit.jupiter.params.ParameterizedTest; import javax.annotation.Nonnull; +import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -802,6 +805,57 @@ public void counterBasedSubspaceKeysProtoSettings() { assertEquals("Error converting from protobuf", e.getMessage()); } + @Test + void storedRecordTypesOfSyntheticRecordTypes() { + final RecordMetaData metaData = syntheticRecordTypesMetaData(); + + final RecordType outer = metaData.getRecordType("OuterRecord"); + final RecordType other = metaData.getRecordType("OtherRecord"); + assertEquals(Set.of(outer, other), metaData.getSyntheticRecordType("OuterOtherJoined").getStoredRecordTypes()); + // A type joined to itself is a single stored type. + assertEquals(Set.of(outer), metaData.getSyntheticRecordType("SelfJoined").getStoredRecordTypes()); + // The nested constituent is a message within the parent, not a stored type. + assertEquals(Set.of(outer), metaData.getSyntheticRecordType("UnnestedMap").getStoredRecordTypes()); + } + + @Test + void syntheticRecordTypesFromStoredRecordTypes() { + final RecordMetaData metaData = syntheticRecordTypesMetaData(); + + final RecordType outer = metaData.getRecordType("OuterRecord"); + final RecordType other = metaData.getRecordType("OtherRecord"); + assertEquals(Set.of("OuterOtherJoined", "SelfJoined", "UnnestedMap"), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer, other)))); + assertEquals(Set.of("SelfJoined", "UnnestedMap"), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(outer)))); + assertEquals(Set.of(), + syntheticRecordTypeNames(metaData.getSyntheticRecordTypesFromRecordTypes(Set.of(other)))); + } + + @Nonnull + private static RecordMetaData syntheticRecordTypesMetaData() { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsNestedMapProto.getDescriptor()); + final JoinedRecordTypeBuilder joined = metaDataBuilder.addJoinedRecordType("OuterOtherJoined"); + joined.addConstituent("outer", "OuterRecord"); + joined.addConstituent("other", "OtherRecord"); + joined.addJoin("outer", field("other_id"), "other", field("other_id")); + final JoinedRecordTypeBuilder selfJoined = metaDataBuilder.addJoinedRecordType("SelfJoined"); + selfJoined.addConstituent("first", "OuterRecord"); + selfJoined.addConstituent("second", "OuterRecord"); + selfJoined.addJoin("first", field("other_id"), "second", field("rec_id")); + final UnnestedRecordTypeBuilder unnested = metaDataBuilder.addUnnestedRecordType("UnnestedMap"); + unnested.addParentConstituent("parent", metaDataBuilder.getRecordType("OuterRecord")); + unnested.addNestedConstituent("entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + "parent", field("map").nest(field("entry", KeyExpression.FanType.FanOut))); + return metaDataBuilder.build(); + } + + @Nonnull + private static Set syntheticRecordTypeNames(@Nonnull final Collection> syntheticRecordTypes) { + return syntheticRecordTypes.stream().map(RecordType::getName).collect(Collectors.toSet()); + } + @Test void canSerializeAndDeserializeSyntheticRecordTypes() { RecordMetaDataBuilder rmd = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());