synthetic joined record type planning

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

@@ -949,8 +949,10 @@
                 relationalExpression = childCompensation.apply(memoizer, relationalExpression,
                         matchedToRealizedTranslationMapFunction);
             }
 
-            final var matchedForEachAlias = getMatchedForEachAlias();
+            // The compensating predicates are expressed over the top of the match, so the alias picked here only names
+            // the quantifier they are re-targeted onto -- any matched for-each alias will do, and a join match has several.
+            final var matchedForEachAlias = Iterables.get(getMatchedForEachAliases(), 0);
             final var matchedToRealizedTranslationMap = matchedToRealizedTranslationMapFunction.apply(matchedForEachAlias);
 
             final var compensatedPredicates = new LinkedIdentitySet<QueryPredicate>();
@@ -1036,22 +1038,22 @@
             }
         }
 
+        /**
+         * {@return the aliases of the matched for-each quantifiers}
+         */
         @Nonnull
-        private CorrelationIdentifier getMatchedForEachAlias() {
+        private Set<CorrelationIdentifier> getMatchedForEachAliases() {
             final var matchedQuantifierMap =
                     Quantifiers.aliasToQuantifierMap(matchedQuantifiers);
 
             final var matchedAliases =
                     matchedQuantifierMap.keySet();
             Verify.verify(compensatedAliases.equals(matchedAliases));
 
-            final var matchedForEachQuantifierAliases =
-                    matchedAliases
-                            .stream()
-                            .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach)
-                            .collect(ImmutableSet.toImmutableSet());
-
-            return Iterables.getOnlyElement(matchedForEachQuantifierAliases);
+            return matchedAliases
+                    .stream()
+                    .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach)
+                    .collect(ImmutableSet.toImmutableSet());
         }
 
         @Nonnull
@@ -1062,7 +1064,9 @@
             Verify.verify(!isImpossible());
             Verify.verify(resultCompensationFunction.isNeeded());
 
-            final var matchedForEachAlias = getMatchedForEachAlias();
+            // The compensated result value is expressed over the top of the match, so the alias picked here only names
+            // the quantifier it is re-targeted onto -- any matched for-each alias will do, and a join match has several.
+            final var matchedForEachAlias = Iterables.get(getMatchedForEachAliases(), 0);
 
             final var resultValue =
                     resultCompensationFunction.applyCompensationForResult(

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java

@@ -38,7 +38,8 @@
 import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges;
 import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.ProtoUtils;
 import com.google.common.base.Verify;
@@ -445,7 +446,37 @@
     @Nonnull
     @Override
     public GraphExpansion visitExpression(@Nonnull final ListKeyExpression listKeyExpression) {
-        throw new UnsupportedOperationException("visitor method for this key expression is not implemented");
+        final ImmutableList.Builder<GraphExpansion> expandedPredicatesBuilder = ImmutableList.builder();
+        final VisitorState state = getCurrentState();
+        int currentOrdinal = state.getCurrentOrdinal();
+        for (KeyExpression child : listKeyExpression.getChildren()) {
+            final VisitorState childState = state.withCurrentOrdinal(currentOrdinal);
+            final GraphExpansion childExpansion = pop(child.expand(push(childState.forFunctionalExpansion())));
+            if (!childExpansion.getQuantifiers().isEmpty()) {
+                throw new UnsupportedOperationException("cannot expand a list whose child introduces quantifiers");
+            }
+            final var childValues = childExpansion.getResultColumns()
+                    .stream()
+                    .map(Column::getValue)
+                    .collect(ImmutableList.toImmutableList());
+            // a list places its child in a nested tuple, so the value of the position is the record of the child's
+            // column values -- a record of one for a single-column child, as that tuple is nested all the same
+            final var value = childState.registerValue(RecordConstructorValue.ofUnnamed(childValues));
+            // the child's position contributes what a scalar field at that position would
+            final var isSargable = childState.isKey() && !childState.isInternalExpansion();
+            if (childState.isSelectStar()) {
+                expandedPredicatesBuilder.add(isSargable
+                                              ? GraphExpansion.ofPlaceholder(value.asPlaceholder(newParameterAlias()))
+                                              : GraphExpansion.empty());
+            } else {
+                expandedPredicatesBuilder.add(isSargable
+                                              ? GraphExpansion.ofResultColumnAndPlaceholder(Column.unnamedOf(value),
+                                                      value.asPlaceholder(newParameterAlias()))
+                                              : GraphExpansion.ofResultColumn(Column.unnamedOf(value)));
+            }
+            currentOrdinal++;
+        }
+        return GraphExpansion.ofOthers(expandedPredicatesBuilder.build());
     }
 
     /**

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java

@@ -33,7 +33,8 @@
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.ProtoUtils;
 import com.google.common.collect.ImmutableList;
@@ -206,7 +207,21 @@
     @Nonnull
     @Override
     public Value visitExpression(@Nonnull final ListKeyExpression listKeyExpression) {
-        throw new UnsupportedOperationException("visitor method for this key expression is not implemented");
+        if (listKeyExpression.getColumnSize() > 1) {
+            throw new RecordCoreException("cannot expand ListKeyExpression in scalar expansion");
+        }
+
+        final ScalarVisitorState state = getCurrentState();
+        final KeyExpression child = Iterables.getOnlyElement(listKeyExpression.getChildren());
+
+        // The child occupies a single position whose value is the nested tuple of the child's own columns, so it is a
+        // record of their values -- a record of one for a single-column child, as that tuple is nested all the same.
+        final List<KeyExpression> positions = child.normalizeKeyForPositions();
+        final ImmutableList.Builder<Value> valuesBuilder = ImmutableList.builder();
+        for (final KeyExpression position : positions) {
+            valuesBuilder.add(pop(position.expand(push(state))));
+        }
+        return RecordConstructorValue.ofUnnamed(valuesBuilder.build());
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java

@@ -149,18 +149,21 @@
         final var logicalKeyValuesBuilder = ImmutableList.<Value>builder();
         for (int i = 0; i < indexKeyValues.size(); i++) {
             final Value keyValue = indexKeyValues.get(i);
 
-            final var extractFromIndexEntryPairOptional =
-                    keyValue.extractFromIndexEntryMaybe(baseObjectValue, EvaluationContext.empty(), AliasMap.emptyMap(),
-                            ImmutableSet.of(), IndexKeyValueToPartialRecord.TupleSource.KEY, ImmutableIntArray.of(i));
-            if (extractFromIndexEntryPairOptional.isPresent()) {
-                final var extractFromIndexEntryPair = extractFromIndexEntryPairOptional.get();
-                if (!addCoveringField(builder, extractFromIndexEntryPair.getKey(),
-                        extractFromIndexEntryPair.getValue())) {
-                    return Optional.empty();
+            for (final var datum : entryData(keyValue, ImmutableIntArray.of(i))) {
+                final var extractFromIndexEntryPairOptional =
+                        datum.getLeft().extractFromIndexEntryMaybe(baseObjectValue, EvaluationContext.empty(),
+                                AliasMap.emptyMap(), ImmutableSet.of(),
+                                IndexKeyValueToPartialRecord.TupleSource.KEY, datum.getRight());
+                if (extractFromIndexEntryPairOptional.isPresent()) {
+                    final var extractFromIndexEntryPair = extractFromIndexEntryPairOptional.get();
+                    if (!addCoveringField(builder, extractFromIndexEntryPair.getKey(),
+                            extractFromIndexEntryPair.getValue())) {
+                        return Optional.empty();
+                    }
+                    recordCoveredField(covered, extractFromIndexEntryPair);
+                    logicalKeyValuesBuilder.add(extractFromIndexEntryPair.getLeft());
                 }
-                recordCoveredField(covered, extractFromIndexEntryPair);
-                logicalKeyValuesBuilder.add(extractFromIndexEntryPair.getLeft());
             }
         }
 
@@ -192,6 +195,27 @@
                         indexEntryToRecordValue(baseType, covered)));
     }
 
+    /**
+     * Decomposes one column of the index entry into the data it holds that can be read individually.
+     *
+     * @param value the value registered for the column
+     * @param ordinalPath the path to the column
+     * @return the data that column holds, each datum paired with the path at which it sits in the entry's tuple
+     */
+    @Nonnull
+    private static List<NonnullPair<Value, ImmutableIntArray>> entryData(@Nonnull final Value value,
+                                                                         @Nonnull final ImmutableIntArray ordinalPath) {
+        if (!(value instanceof RecordConstructorValue)) {
+            return ImmutableList.of(NonnullPair.of(value, ordinalPath));
+        }
+        final var columns = ((RecordConstructorValue)value).getColumns();
+        final var dataBuilder = ImmutableList.<NonnullPair<Value, ImmutableIntArray>>builder();
+        for (int i = 0; i < columns.size(); i++) {
+            dataBuilder.addAll(entryData(columns.get(i).getValue(), ImmutableIntArray.builder().addAll(ordinalPath).add(i).build()));
+        }
+        return dataBuilder.build();
+    }
+
     /**
      * Records what the given extraction covers, descending a node per field of the path it fills, which may run into
      * nested messages. Every name is present here, {@code addCoveringField} having already refused the extraction

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java

@@ -23,7 +23,8 @@
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.metadata.IndexTypes;
 import com.apple.foundationdb.record.metadata.RecordType;
+import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.KeyWithValueExpression;
@@ -32,9 +33,12 @@
 import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 
 import javax.annotation.Nonnull;
@@ -43,6 +47,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 import static com.apple.foundationdb.record.metadata.Key.Expressions.concat;
@@ -72,6 +77,62 @@
         this.queriedRecordTypes = ImmutableList.copyOf(queriedRecordTypes);
     }
 
+    /**
+     * Returns the {@link SyntheticRecordType} an index is defined on, if it is defined on exactly one.
+     */
+    @Nonnull
+    private static Optional<SyntheticRecordType<?>> syntheticRecordTypeMaybe(
+            @Nonnull final Collection<RecordType> indexedRecordTypes) {
+        if (indexedRecordTypes.size() != 1) {
+            return Optional.empty();
+        }
+        final RecordType indexedRecordType = Iterables.getOnlyElement(indexedRecordTypes);
+        return indexedRecordType instanceof final SyntheticRecordType<?> syntheticRecordType
+                ? Optional.of(syntheticRecordType)
+                : Optional.empty();
+    }
+
+    /**
+     * Merges the expansion of an index key into the expansion of the synthetic type the index is defined on, if it is
+     * defined on one. An index on a stored type has no base expansion, and its key expansion is returned as it stands.
+     *
+     * <p>The key's values are written over {@code baseQuantifier}, which does not survive into the merged graph, so
+     * each is re-expressed over the record the base expansion assembles and simplified:
+     * {@code base.customer.name} over {@code (c AS customer, o AS orders)} becomes {@code c.name}, the very value a
+     * query performing the same join flows.
+     *
+     * <p>Predicates and placeholders are translated separately because a placeholder is held in both lists and
+     * {@link GraphExpansion#seal()} pairs them up by value; translating one alone would leave a parameter whose
+     * predicate the select does not hold.
+     *
+     * @param baseExpansion the expansion assembling the synthetic type's records, or {@code null} if there is none
+     * @param keyExpansion the expansion of the index key, expressed over {@code baseQuantifier}
+     * @param baseQuantifier the quantifier {@code keyExpansion} is expressed over
+     * @return the key expansion merged into the base expansion, or the key expansion itself if there is no base
+     */
+    @Nonnull
+    private static GraphExpansion mergeWithBaseIfNeeded(@Nullable final GraphExpansion baseExpansion,
+                                                        @Nonnull final GraphExpansion keyExpansion,
+                                                        @Nonnull final Quantifier.ForEach baseQuantifier) {
+        if (baseExpansion == null) {
+            return keyExpansion;
+        }
+        final var ontoBaseRecord = TranslationMap.regularBuilder()
+                .when(baseQuantifier.getAlias())
+                .then((sourceAlias, leafValue) -> RecordConstructorValue.ofColumns(baseExpansion.getResultColumns()))
+                .build();
+        return GraphExpansion.ofOthers(
+                baseExpansion.toBuilder().removeAllResultColumns().build(),
+                keyExpansion.toBuilder()
+                        .removeAllPredicates()
+                        .addAllPredicates(keyExpansion.getPredicates().stream()
+                                .map(predicate -> predicate.translateCorrelations(ontoBaseRecord, true))
+                                .collect(ImmutableList.toImmutableList()))
+                        .replacePlaceholder(placeholder ->
+                                placeholder.translateLeafPredicate(ontoBaseRecord, true))
+                        .build());
+    }
+
     @Nonnull
     @Override
     public MatchCandidate expand(@Nonnull final Set<String> availableRecordTypeNames,
@@ -87,12 +148,22 @@
         // the instantiation of the type filter below to create a placeholder for the record type key parameter
         // alias and reuse it here. Similar to what we currently do for primary scans.
         //
-        final var baseQuantifier = Quantifier.forEach(ExpansionVisitor.createBaseRef(availableRecordTypeNames,
-                queriedRecordTypeNames, baseType, null, accessHint));
+        @Nullable final GraphExpansion baseExpansion =
+                syntheticRecordTypeMaybe(queriedRecordTypes).map(recordType -> recordType.expand(accessHint)).orElse(null);
         final var allExpansionsBuilder = ImmutableList.<GraphExpansion>builder();
-
-        // add the value for the flow of records
-        allExpansionsBuilder.add(GraphExpansion.ofQuantifier(baseQuantifier));
+        final Quantifier.ForEach baseQuantifier;
+        final Value baseObjectValue;
+        if (baseExpansion == null) {
+            baseQuantifier = Quantifier.forEach(ExpansionVisitor.createBaseRef(availableRecordTypeNames,
+                    queriedRecordTypeNames, baseType, null, accessHint));
+            baseObjectValue = baseQuantifier.getFlowedObjectValue();
+            // add the value for the flow of records
+            allExpansionsBuilder.add(GraphExpansion.ofQuantifier(baseQuantifier));
+        } else {
+            // the base expansion's own quantifiers reach the graph when the key expansion is merged into it
+            baseQuantifier = Quantifier.forEach(Reference.initialOf(baseExpansion.seal().buildSelect()));
+            baseObjectValue = RecordConstructorValue.ofColumns(baseExpansion.getResultColumns());
+        }
 
         var rootExpression = index.getRootExpression();
 
@@ -105,8 +176,7 @@
         }
 
         final int keyValueSplitPoint;
-        if (rootExpression instanceof KeyWithValueExpression) {
-            final KeyWithValueExpression keyWithValueExpression = (KeyWithValueExpression)rootExpression;
+        if (rootExpression instanceof final KeyWithValueExpression keyWithValueExpression) {
             keyValueSplitPoint = keyWithValueExpression.getSplitPoint();
             rootExpression = keyWithValueExpression.getInnerKey();
         } else {
@@ -190,15 +260,18 @@
             }
         }
 
-        final var completeExpansion = GraphExpansion.ofOthers(allExpansionsBuilder.build());
-        final var sealedExpansion = completeExpansion.seal();
+        final var sealedExpansion = mergeWithBaseIfNeeded(
+                baseExpansion,
+                GraphExpansion.ofOthers(allExpansionsBuilder.build()),
+                baseQuantifier)
+                .seal();
         final var parameters =
                 sealedExpansion.getPlaceholders()
                         .stream()
                         .map(Placeholder::getParameterAlias)
                         .collect(ImmutableList.toImmutableList());
         final var matchableSortExpression = new MatchableSortExpression(parameters, isReverse,
-                sealedExpansion.buildSelectWithResultValue(baseQuantifier.getFlowedObjectValue()));
+                sealedExpansion.buildSelectWithResultValue(baseObjectValue));
         return new ValueIndexScanMatchCandidate(index,
                 queriedRecordTypes,
                 Traversal.withRoot(Reference.initialOf(matchableSortExpression)),