Skip to content

Cost-free covering scans for values computed from index columns #4875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
hatyo wants to merge 1 commit into
base: main
Choose a base branch
from
+1,941 −1,567
Draft

Cost-free covering scans for values computed from index columns #4875

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...java/com/apple/foundationdb/record/query/plan/cascades/IndexEntryToRecordValueHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public RecordConstructorValue toRecordValue(@Nonnull final Type.Record targetTyp
* array, which cannot be null and whose emptiness is indistinguishable from being unset.
*/
@Nonnull
private static Value absent(@Nonnull final Type fieldType) {
static Value absent(@Nonnull final Type fieldType) {
if (fieldType.getTypeCode() == Type.TypeCode.ARRAY && !fieldType.isNullable()) {
return LightArrayConstructorValue.emptyArray(
Objects.requireNonNull(((Type.Array)fieldType).getElementType()));
Expand Down
114 changes: 73 additions & 41 deletions ...n/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
import com.apple.foundationdb.record.query.plan.cascades.values.translation.MaxMatchMap;
import com.apple.foundationdb.record.util.pair.NonnullPair;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
Expand All @@ -54,47 +55,21 @@ static Optional<Value> pushValueThroughFetch(@Nonnull final Value toBePushedValu
@Nonnull final CorrelationIdentifier baseAlias,
@Nonnull final CorrelationIdentifier sourceAlias,
@Nonnull final CorrelationIdentifier targetAlias,
@Nonnull final Iterable<? extends Value> providedValuesFromIndex) {
if (!isOfPushableTypesOrConstant(toBePushedValue, sourceAlias)) {
return Optional.empty();
}

final AliasMap equivalenceMap = AliasMap.ofAliases(sourceAlias, baseAlias);
final AliasMap toTargetAliasMap = AliasMap.ofAliases(sourceAlias, targetAlias);

final var translatedValueOptional =
toBePushedValue.<Value>mapMaybe((value, mappedChildren) -> {
for (final var providedValue : providedValuesFromIndex) {
if (value.semanticEquals(providedValue, equivalenceMap)) {
return value.withChildren(mappedChildren)
.rebase(toTargetAliasMap);
}
}
return value.withChildren(mappedChildren); // this may be correlated to sourceAlias
});
@Nonnull final Value providedRecordValue) {
//
// Maximum-match the value against the record the index scan provides, so that as much of it as the entry can
// supply gets expressed over the scan. A value computed from provided fields pushes even though it is not
// itself a field, while one reading a field the entry lacks stays correlated to sourceAlias and fails below.
//
final var maxMatchMap =
MaxMatchMap.compute(toBePushedValue, providedRecordValue,
ImmutableSet.of(baseAlias),
ValueEquivalence.fromAliasMap(AliasMap.ofAliases(sourceAlias, baseAlias)));

// the translation was successful if the translated value is not correlated to sourceAlias anymore
return translatedValueOptional.filter(translatedValue -> !translatedValue.getCorrelatedTo().contains(sourceAlias));
}

private static boolean isOfPushableTypesOrConstant(@Nonnull final Value toBePushedValue,
@Nonnull final CorrelationIdentifier sourceAlias) {
if (!toBePushedValue.getCorrelatedTo().contains(sourceAlias)) {
return true;
}
if (toBePushedValue instanceof FieldValue) {
return true;
} else if (toBePushedValue instanceof RecordConstructorValue) {
return ((RecordConstructorValue)toBePushedValue).getColumns().stream()
.allMatch(column -> isOfPushableTypesOrConstant(column.getValue(), sourceAlias));
} else {
// Effectively, this check is needed because of values like the VersionValue, which aren't
// accessible without a record fetch, even if the index entry contains a VersionValue. We
// should address this by attaching the version to a partial record if there's one in the
// index entry, but until then, this prevents us from incorrectly applying a covering
// optimization.
return false;
}
return maxMatchMap.translateQueryValueMaybe(targetAlias)
.map(translatedValue -> translatedValue.simplify(EvaluationContext.empty(), AliasMap.emptyMap(), ImmutableSet.of()))
.filter(translatedValue -> !translatedValue.getCorrelatedTo().contains(sourceAlias));
}

static boolean addCoveringField(@Nonnull final IndexKeyValueToPartialRecord.Builder builder,
Expand Down Expand Up @@ -146,6 +121,7 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
final var builder = IndexKeyValueToPartialRecord.newBuilder(queriedRecordType);
final var baseObjectValue = QuantifiedObjectValue.of(baseAlias, baseType);
final var covered = new IndexEntryToRecordValueHelper();
final var providedFieldsBuilder = ImmutableList.<FieldValue>builder();
final var logicalKeyValuesBuilder = ImmutableList.<Value>builder();
for (int i = 0; i < indexKeyValues.size(); i++) {
final Value keyValue = indexKeyValues.get(i);
Expand All @@ -160,6 +136,7 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
return Optional.empty();
}
recordCoveredField(covered, extractFromIndexEntryPair);
providedFieldsBuilder.add(extractFromIndexEntryPair.getKey());
logicalKeyValuesBuilder.add(extractFromIndexEntryPair.getLeft());
}
}
Expand All @@ -178,6 +155,7 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
return Optional.empty();
}
recordCoveredField(covered, extractFromIndexEntryPair);
providedFieldsBuilder.add(extractFromIndexEntryPair.getKey());
logicalValueValuesBuilder.add(extractFromIndexEntryPair.getLeft());
}
}
Expand All @@ -189,7 +167,58 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
return Optional.of(
new ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord(queriedRecordType, builder.build(),
logicalKeyValuesBuilder.build(), logicalValueValuesBuilder.build(),
indexEntryToRecordValue(baseType, covered)));
indexEntryToRecordValue(baseType, covered),
providedRecordValue(baseType, providedFieldsBuilder.build())));
}

/**
* The record a covering scan provides, as a value over the base: every field the entry carries reads itself, the
* rest read absent. Addressed by ordinal rather than by field name, because the names an index's extracted
* {@link FieldValue}s carry are the record descriptor's, which differ from the base type's wherever an identifier
* had to be escaped.
*
* @param baseType the record an entry is read into
* @param providedFields what the entry carries, as paths into {@code baseType}
*
* @return that value, or {@code null} if one cannot be built
*/
@Nullable
private static Value providedRecordValue(@Nonnull final Type baseType,
@Nonnull final List<FieldValue> providedFields) {
if (!(baseType instanceof Type.Record)) {
return null;
}
return providedRecordValue((Type.Record)baseType, providedFields, 0);
}

@Nonnull
private static Value providedRecordValue(@Nonnull final Type.Record recordType,
@Nonnull final List<FieldValue> providedFields,
final int depth) {
final var fields = recordType.getFields();
final var columns = ImmutableList.<Column<? extends Value>>builder();
for (int ordinal = 0; ordinal < fields.size(); ordinal++) {
final var field = fields.get(ordinal);
final int currentOrdinal = ordinal;
final var through = providedFields.stream()
.filter(fieldValue -> fieldValue.getFieldOrdinals().length() > depth
&& fieldValue.getFieldOrdinals().get(depth) == currentOrdinal)
.collect(ImmutableList.toImmutableList());
// the first one ending here wins, keeping the first of an index's repeated columns for one field
final var endingHere = through.stream()
.filter(fieldValue -> fieldValue.getFieldOrdinals().length() == depth + 1)
.findFirst();
final Value column;
if (endingHere.isPresent()) {
column = endingHere.get();
} else if (!through.isEmpty() && field.getFieldType() instanceof Type.Record) {
column = providedRecordValue((Type.Record)field.getFieldType(), through, depth + 1);
} else {
column = IndexEntryToRecordValueHelper.absent(field.getFieldType());
}
columns.add(Column.of(field, column));
}
return RecordConstructorValue.ofColumns(columns.build());
}

/**
Expand Down Expand Up @@ -228,10 +257,13 @@ private static Value indexEntryToRecordValue(@Nonnull final Type baseType,
* @param logicalKeyValues what each entry key column reads from the record
* @param logicalValueValues what each entry value column reads from the record
* @param indexEntryToRecordValue the value that decodes it, or {@code null} if one cannot be built
* @param providedRecordValue the record a covering scan provides, each carried field reading itself, or
* {@code null} if one cannot be built
*/
record IndexEntryToLogicalRecord(@Nonnull RecordType queriedRecordType,
@Nonnull IndexKeyValueToPartialRecord indexKeyValueToPartialRecord,
@Nonnull List<Value> logicalKeyValues, @Nonnull List<Value> logicalValueValues,
@Nullable Value indexEntryToRecordValue) {
@Nullable Value indexEntryToRecordValue,
@Nullable Value providedRecordValue) {
}
}
8 changes: 5 additions & 3 deletions .../java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -289,13 +288,16 @@ public Optional<Value> pushValueThroughFetch(@Nonnull final Value toBePushedValu
@Nonnull final CorrelationIdentifier targetAlias) {
final var indexEntryToLogicalRecord =
getIndexEntryToLogicalRecordMaybe().orElseThrow(() -> new RecordCoreException("need index entry to logical record"));
final var providedRecordValue = indexEntryToLogicalRecord.providedRecordValue();
if (providedRecordValue == null) {
return Optional.empty();
}

return ScanWithFetchMatchCandidate.pushValueThroughFetch(toBePushedValue,
baseAlias,
sourceAlias,
targetAlias,
Iterables.concat(indexEntryToLogicalRecord.logicalKeyValues(),
indexEntryToLogicalRecord.logicalValueValues()));
providedRecordValue);
}

@Override
Expand Down
6 changes: 5 additions & 1 deletion ...va/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,12 +457,16 @@ public Optional<Value> pushValueThroughFetch(@Nonnull Value toBePushedValue,
@Nonnull CorrelationIdentifier targetAlias) {
final var indexEntryToLogicalRecord =
getIndexEntryToLogicalRecordMaybe().orElseThrow(() -> new RecordCoreException("need index entry to logical record"));
final var providedRecordValue = indexEntryToLogicalRecord.providedRecordValue();
if (providedRecordValue == null) {
return Optional.empty();
}

return ScanWithFetchMatchCandidate.pushValueThroughFetch(toBePushedValue,
baseAlias,
sourceAlias,
targetAlias,
indexEntryToLogicalRecord.logicalKeyValues());
providedRecordValue);
}

@Nonnull
Expand Down
31 changes: 17 additions & 14 deletions ...e/foundationdb/record/provider/foundationdb/query/FDBLongArithmeticFunctionQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,16 +448,19 @@ void matchConstantMaskValue() {
assertEquals(-1043926565, plan.planHash(PlanHashable.CURRENT_LEGACY));
assertEquals(-1859939974, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION));
} else {
// The other mask values do not work, and so they require executing as a scan
// The other mask values do not match the literal in the expected index, but both the predicate
// and the result can be computed from the entries of the index on num_value_unique, so the scan
// over that index stays covering and the mask is applied as a residual predicate.
assertMatchesExactly(plan, mapPlan(
predicatesFilterPlan(
typeFilterPlan(
scanPlan().where(scanComparisons(unbounded())
)
coveringIndexPlan().where(indexPlanOf(
indexPlan().where(indexName("MySimpleRecord$num_value_unique"))
.and(scanComparisons(unbounded()))
))
)
)));
// assertEquals(0, plan.planHash(PlanHashable.CURRENT_LEGACY));
assertEquals(-1713033856, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION));
));
// ConstantObjectValue does not support the legacy plan hash mode
assertEquals(-1086853773, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION));
}

for (long queryMaskValue : List.of(0L, mask, mask + 1)) {
Expand Down Expand Up @@ -512,15 +515,15 @@ void calculateFunctionFromCoveringIndexScan() {
Quantifier selectQun = Quantifier.forEach(Reference.initialOf(select));
return Reference.initialOf(new LogicalSortExpression(RequestedOrdering.preserve(), selectQun));
});
// This should be planned as a covering index scan of the index because the functions can be calculated from the values
// in the index. Until matching improves, we can get by with this plan
// This is planned as a covering index scan of the index because the functions can be calculated from the
// values in the index.
assertMatchesExactly(plan, mapPlan(
typeFilterPlan(
scanPlan().where(scanComparisons(unbounded()))
)
coveringIndexPlan().where(indexPlanOf(
indexPlan().where(indexName(index.getName())).and(scanComparisons(unbounded()))
))
));
assertEquals(1263343956, plan.planHash(PlanHashable.CURRENT_LEGACY));
assertEquals(-2029074143, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION));
assertEquals(1905107315, plan.planHash(PlanHashable.CURRENT_LEGACY));
assertEquals(355706836, plan.planHash(PlanHashable.CURRENT_FOR_CONTINUATION));

Set<Map<String, Object>> results = new HashSet<>();
try (RecordCursor<QueryResult> cursor = executeCascades(recordStore, plan)) {
Expand Down
Loading
Loading