diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java index 4b0ee0463e2a..46f39cfcfa4c 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java @@ -663,6 +663,18 @@ public static Integer getMaxRowsInWindow(Map queryOptions) { return checkedParseIntPositive(QueryOptionKey.MAX_ROWS_IN_WINDOW, maxRowsInWindow); } + @Nullable + public static Integer getMaxRowsInMatchPartition(Map queryOptions) { + String maxRowsInMatchPartition = queryOptions.get(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION); + return checkedParseIntPositive(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, maxRowsInMatchPartition); + } + + @Nullable + public static Long getMaxStepsPerMatchAttempt(Map queryOptions) { + String maxStepsPerMatchAttempt = queryOptions.get(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT); + return checkedParseLongPositive(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, maxStepsPerMatchAttempt); + } + @Nullable public static WindowOverFlowMode getWindowOverflowMode(Map queryOptions) { String windowOverflowModeStr = queryOptions.get(QueryOptionKey.WINDOW_OVERFLOW_MODE); diff --git a/pinot-common/src/main/proto/expressions.proto b/pinot-common/src/main/proto/expressions.proto index cd185eba0843..b24b29b34180 100644 --- a/pinot-common/src/main/proto/expressions.proto +++ b/pinot-common/src/main/proto/expressions.proto @@ -52,6 +52,30 @@ message InputRef { int32 index = 1; } +// A reference to a column of the row bound to a specific MATCH_RECOGNIZE pattern variable, as it appears inside +// MEASURES and DEFINE expressions. Mirrors Calcite's RexPatternFieldRef, which carries the pattern variable +// alongside the input index. +// +// This is deliberately NOT modelled as an InputRef. RexPatternFieldRef extends RexInputRef, so anything that only +// looks at the index silently turns `DEFINE UP AS UP.price > PREV(UP.price)` into a read of the current row's +// column: results that are wrong but still type-correct, and therefore invisible. Any consumer that does not +// understand this expression case must fail loudly instead of degrading it to an InputRef. +// +// Only meaningful inside a MatchNode: symbolOrdinal indexes MatchNode.patternSymbols. +message PatternFieldRef { + // Column index into the MatchNode input row. + int32 index = 1; + // Ordinal of the pattern variable this reference is bound to, i.e. the index into MatchNode.patternSymbols. + // The sentinel -2 denotes the SQL:2016 universal row pattern variable: an unqualified reference that ranges over + // every row in the match regardless of its pattern variable. -1 is reserved for an unresolved planner-local + // reference and must never reach the wire; values below -2 are invalid. This field is authoritative. + int32 symbolOrdinal = 2; + // Pattern variable name as written in the query, carried for explain plans and error messages only. For the + // universal ordinal (-2), Calcite supplies the row-source/table alias here; it is not a pattern-variable name. + // Consumers must resolve the variable through symbolOrdinal and never string-match on this field. + string alpha = 3; +} + message Literal { ColumnDataType dataType = 1; oneof value { @@ -108,5 +132,6 @@ message Expression { InputRef inputRef = 1; Literal literal = 2; FunctionCall functionCall = 3; + PatternFieldRef patternFieldRef = 4; } } diff --git a/pinot-common/src/main/proto/plan.proto b/pinot-common/src/main/proto/plan.proto index f1f1ef9dba7e..fe6c7fbb874b 100644 --- a/pinot-common/src/main/proto/plan.proto +++ b/pinot-common/src/main/proto/plan.proto @@ -44,6 +44,10 @@ message PlanNode { // serialized by an older-version broker can still be deserialized. No current broker sets this field. EnrichedJoinNode enrichedJoinNode = 17 [deprecated = true]; UnnestNode unnestNode = 18; + // Rolling-upgrade boundary: servers released before MATCH_RECOGNIZE do not know field 19. Protobuf preserves it + // as an unknown field, but their PlanNodeDeserializer sees NODE_NOT_SET and cannot execute the stage. Upgrade all + // servers before issuing MATCH_RECOGNIZE queries; there is no mixed-version fallback to another node kind. + MatchNode matchNode = 19; } } @@ -264,6 +268,119 @@ message UnnestNode { bool prunedPassthrough = 6; } +// --------------------------------------------------------------------------------------------------------------- +// MATCH_RECOGNIZE (SQL:2016 row pattern recognition) +// --------------------------------------------------------------------------------------------------------------- + +// SQL:2016 `AFTER MATCH SKIP` clause: where pattern matching resumes after a successful match. +// +// SKIP_PAST_LAST_ROW is 0 on purpose. It is the SQL:2016 default (and the default in Trino, Snowflake and Oracle), +// so a MatchNode that omits the field falls back to non-overlapping matches rather than to the overlapping +// SKIP TO NEXT ROW semantics that Calcite defaults to internally when the clause is omitted. +// +// The values are wire-stable; renumbering them silently changes query results across mixed-version brokers/servers. +enum AfterMatchSkipMode { + SKIP_PAST_LAST_ROW = 0; + SKIP_TO_NEXT_ROW = 1; + SKIP_TO_FIRST = 2; + SKIP_TO_LAST = 3; +} + +// SQL:2016 `ONE ROW PER MATCH` / `ALL ROWS PER MATCH`. ALL_ROWS_PER_MATCH is declared so that its wire value is +// pinned, but it is rejected at planning time until per-row output is implemented. +enum RowsPerMatchMode { + ONE_ROW_PER_MATCH = 0; + ALL_ROWS_PER_MATCH = 1; +} + +// The kind of a RowPattern tree node. PATTERN_EXCLUDE and PATTERN_PERMUTE are declared up front so the deferred +// work cannot renumber the supported kinds; a plan carrying them is rejected at deserialization time. +enum RowPatternKind { + // A single pattern variable. Carries symbolOrdinal. + PATTERN_SYMBOL = 0; + // Juxtaposition, e.g. `A B C`. Two or more children, in pattern order. + PATTERN_CONCAT = 1; + // Alternation `|`, e.g. `A | B`. Two or more children, in pattern order (leftmost alternative wins). + PATTERN_ALTERNATE = 2; + // A quantifier applied to exactly one child. Carries quantifier. + PATTERN_QUANTIFIER = 3; + // The `^` anchor: the match must start at the first row of the partition. No children. + PATTERN_ANCHOR_START = 4; + // The `$` anchor: the match must end at the last row of the partition. No children. + PATTERN_ANCHOR_END = 5; + // Deferred: `{- ... -}` exclusion. Exactly one child. + PATTERN_EXCLUDE = 6; + // Deferred: `PERMUTE(...)`. Two or more children. + PATTERN_PERMUTE = 7; +} + +// A `{n,m}` style repetition applied to the single child of a PATTERN_QUANTIFIER node. `*`, `+` and `?` are +// normalized into this form: `*` is {0,-1}, `+` is {1,-1}, `?` is {0,1}. +message RowPatternQuantifier { + // Minimum number of repetitions, >= 0. + int32 minRepeat = 1; + // Maximum number of repetitions, or -1 for unbounded (`*`, `+`, `{n,}`). + int32 maxRepeat = 2; + // False for the reluctant forms (`*?`, `+?`, `??`, `{n,m}?`), which prefer the shortest match. + bool greedy = 3; +} + +// One node of the row pattern tree. This is a self-contained representation of the PATTERN clause: pattern +// variables are referenced by their ordinal in MatchNode.patternSymbols, so a consumer never string-matches +// variable names. +message RowPattern { + RowPatternKind kind = 1; + // PATTERN_SYMBOL only: ordinal into MatchNode.patternSymbols. + int32 symbolOrdinal = 2; + // PATTERN_CONCAT / PATTERN_ALTERNATE / PATTERN_PERMUTE: two or more children, in pattern order. + // PATTERN_QUANTIFIER / PATTERN_EXCLUDE: exactly one child. Empty for PATTERN_SYMBOL and the anchors. + repeated RowPattern children = 3; + // PATTERN_QUANTIFIER only. + RowPatternQuantifier quantifier = 4; +} + +// One entry of the pattern variable symbol table. The variable's ordinal is its index in +// MatchNode.patternSymbols. +message PatternSymbol { + // Pattern variable name as written in PATTERN / DEFINE. Informational: consumers resolve variables by ordinal. + string name = 1; + // The DEFINE predicate for this variable. Absent when the variable has no DEFINE entry, which per SQL:2016 + // means it matches every row. + optional Expression definition = 2; +} + +// One `MEASURES AS ` item. +message MatchMeasure { + // Output column name. + string name = 1; + Expression expression = 2; +} + +message MatchNode { + // Pattern variable symbol table. A symbol's ordinal is its index in this list; every PATTERN_SYMBOL node and + // every PatternFieldRef refers to a variable by that ordinal. + repeated PatternSymbol patternSymbols = 1; + // Root of the row pattern tree (the PATTERN clause). + RowPattern pattern = 2; + // MEASURES items, in output order. + repeated MatchMeasure measures = 3; + // PARTITION BY: column indexes into the input row. + repeated int32 partitionKeys = 4; + // ORDER BY. Mandatory for MATCH_RECOGNIZE, so this is never empty. + repeated Collation collations = 5; + AfterMatchSkipMode afterMatchSkipMode = 6; + // For SKIP_TO_FIRST / SKIP_TO_LAST: ordinal of the target pattern variable. Absent for the other skip modes. + // Explicitly optional because ordinal 0 is a valid variable, so the proto3 default cannot double as "unset". + optional int32 afterMatchSkipToSymbolOrdinal = 7; + RowsPerMatchMode rowsPerMatchMode = 8; + + // Reserved for the deferred SQL:2016 clauses so that implementing them later cannot renumber anything above: + // 9 -> SUBSET: `repeated PatternSubset subsets` (union variable name -> member symbol ordinals) + // 10 -> WITHIN: the match time interval + reserved 9, 10; + reserved "subsets", "within"; +} + enum WindowFrameType { ROWS = 0; RANGE = 1; diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java index dceac3385eb3..a6024548b53c 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java @@ -38,7 +38,7 @@ public class QueryOptionsUtilsTest { private static final List POSITIVE_INT_KEYS = List.of(NUM_REPLICA_GROUPS_TO_QUERY, MAX_EXECUTION_THREADS, NUM_GROUPS_LIMIT, MAX_INITIAL_RESULT_HOLDER_CAPACITY, - MAX_STREAMING_PENDING_BLOCKS, MAX_ROWS_IN_JOIN, MAX_ROWS_IN_WINDOW); + MAX_STREAMING_PENDING_BLOCKS, MAX_ROWS_IN_JOIN, MAX_ROWS_IN_WINDOW, MAX_ROWS_IN_MATCH_PARTITION); private static final List NON_NEGATIVE_INT_KEYS = List.of(MULTI_STAGE_LEAF_LIMIT); private static final List UNBOUNDED_INT_KEYS = List.of(MIN_SEGMENT_GROUP_TRIM_SIZE, MIN_SERVER_GROUP_TRIM_SIZE, MIN_BROKER_GROUP_TRIM_SIZE, @@ -49,7 +49,8 @@ public class QueryOptionsUtilsTest { addAll(UNBOUNDED_INT_KEYS); }}; private static final List POSITIVE_LONG_KEYS = - List.of(TIMEOUT_MS, MAX_SERVER_RESPONSE_SIZE_BYTES, MAX_QUERY_RESPONSE_SIZE_BYTES); + List.of(TIMEOUT_MS, MAX_SERVER_RESPONSE_SIZE_BYTES, MAX_QUERY_RESPONSE_SIZE_BYTES, + MAX_STEPS_PER_MATCH_ATTEMPT); @Test public void shouldConvertCaseInsensitiveMapToUseCorrectValues() { @@ -333,6 +334,8 @@ private static Object getValue(Map map, String key) { return QueryOptionsUtils.getMaxRowsInJoin(map); case MAX_ROWS_IN_WINDOW: return QueryOptionsUtils.getMaxRowsInWindow(map); + case MAX_ROWS_IN_MATCH_PARTITION: + return QueryOptionsUtils.getMaxRowsInMatchPartition(map); // Non-negative ints case MULTI_STAGE_LEAF_LIMIT: return QueryOptionsUtils.getMultiStageLeafLimit(map); @@ -352,6 +355,8 @@ private static Object getValue(Map map, String key) { return QueryOptionsUtils.getMaxServerResponseSizeBytes(map); case MAX_QUERY_RESPONSE_SIZE_BYTES: return QueryOptionsUtils.getMaxQueryResponseSizeBytes(map); + case MAX_STEPS_PER_MATCH_ATTEMPT: + return QueryOptionsUtils.getMaxStepsPerMatchAttempt(map); default: throw new IllegalArgumentException("Unexpected key!"); } diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/SqlQueryOptionValidationTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/SqlQueryOptionValidationTest.java index fe7cefde8c7d..47c1125d2670 100644 --- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/SqlQueryOptionValidationTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/SqlQueryOptionValidationTest.java @@ -30,6 +30,7 @@ import org.apache.logging.log4j.core.config.Property; import org.apache.pinot.common.utils.config.QueryOptionsUtils; import org.apache.pinot.common.utils.config.QueryOptionsUtils.SqlQueryOptionValidationMode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; @@ -108,6 +109,18 @@ public void rejectModeAcceptsKnownKeysCaseInsensitivelyAndTraceAndDatabase() { assertEquals(options.get("Database"), "db1"); } + @Test + public void matchRecognizeOptionsAreKnownAndCanonicalizedCaseInsensitively() { + QueryOptionsUtils.setSqlQueryOptionValidationMode(SqlQueryOptionValidationMode.REJECT); + + Map options = optionsOf("SET MAXROWSINMATCHPARTITION='100'; " + + "SET maxstepspermatchattempt='200'; SET ALLOWMATCHRECOGNIZEWITHOUTPARTITIONBY='true'; " + + "select * from vegetables"); + assertEquals(options.get(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION), "100"); + assertEquals(options.get(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT), "200"); + assertEquals(options.get(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY), "true"); + } + @Test public void rejectModeLeavesDmlOptionsFreeForm() { QueryOptionsUtils.setSqlQueryOptionValidationMode(SqlQueryOptionValidationMode.REJECT); @@ -165,7 +178,9 @@ public void warnModePreservesUnknownKeysAndLogsOncePerDistinctKey() { // Known keys are never logged. int loggedSoFar = appender.messagesContaining("Unsupported query option").size(); - optionsOf("SET timeoutMs='100'; select * from vegetables"); + optionsOf("SET timeoutMs='100'; SET maxRowsInMatchPartition='200'; " + + "SET maxStepsPerMatchAttempt='300'; SET allowMatchRecognizeWithoutPartitionBy='true'; " + + "select * from vegetables"); assertEquals(appender.messagesContaining("Unsupported query option").size(), loggedSoFar); } finally { appender.detach(); diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MatchRecognizeIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MatchRecognizeIntegrationTest.java new file mode 100644 index 000000000000..4176915093b5 --- /dev/null +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/MatchRecognizeIntegrationTest.java @@ -0,0 +1,690 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.integration.tests.custom; + +import com.fasterxml.jackson.databind.JsonNode; +import java.io.File; +import java.util.List; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.pinot.integration.tests.QueryAssert; +import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.data.Schema; +import org.apache.pinot.spi.utils.builder.TableNameBuilder; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + + +/// End-to-end coverage of SQL:2016 `MATCH_RECOGNIZE` against a real Pinot cluster (ZooKeeper + controller + +/// broker + two servers), as opposed to the mock server enclosures used by +/// `pinot-query-runtime/src/test/resources/queries/MatchRecognize.json`. +/// +/// The fixture is a small stock ticker with five symbols of different lengths, written round-robin into +/// [#getNumAvroFiles()] Avro files, so every partition is spread over several segments and both servers. That +/// makes every assertion here also an assertion that the sort exchange inserted by +/// `PinotMatchExchangeNodeInsertRule` really does deliver each partition contiguously and in ORDER BY order. +/// +/// Every expected result set below is hand-computed from [#PRICES] rather than captured from a run. +@Test(suiteName = "CustomClusterIntegrationTest") +public class MatchRecognizeIntegrationTest extends CustomDataQueryClusterIntegrationTest { + + private static final String DEFAULT_TABLE_NAME = "MatchRecognizeIntegrationTest"; + private static final String SYMBOL_COLUMN = "symbolCol"; + private static final String SEQ_COLUMN = "seqCol"; + private static final String PRICE_COLUMN = "priceCol"; + private static final String NULLABLE_PRICE_COLUMN = "nullablePriceCol"; + + /// Symbols in ascending order, which is also the order every query below sorts its output by. + private static final String[] SYMBOLS = {"AAPL", "AMZN", "GOOG", "MSFT", "NFLX"}; + + /// Prices per symbol, indexed like [#SYMBOLS]. `seqCol` is the 1-based index within the symbol. + /// - AAPL: two disjoint V shapes, the second one overlapping the first under SKIP TO NEXT ROW. + /// - AMZN: one long descent, so a single match covers most of the partition. + /// - GOOG: flat, so no strict rise or fall matches anywhere - a partition that contributes nothing. + /// - MSFT: the minimal V shape. + /// - NFLX: strictly increasing, which is what separates greedy from reluctant quantifiers. + private static final int[][] PRICES = { + {10, 8, 5, 9, 12, 7, 11}, + {20, 15, 10, 5, 25}, + {4, 4, 4}, + {5, 3, 8}, + {1, 2, 3, 4, 5, 6, 7, 8} + }; + + private static final String V_SHAPE_DEFINE = + " DEFINE DOWN AS DOWN.priceCol < PREV(DOWN.priceCol), UP AS UP.priceCol > PREV(UP.priceCol)"; + + /// The canonical vendor-documentation V-shape query: a start row, a strictly falling run, then a strictly rising run. + @Test(dataProvider = "useV2QueryEngine") + public void testCanonicalVShape(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol" + + " ORDER BY seqCol" + + " MEASURES MATCH_NUMBER() AS mno, STRT.seqCol AS start_seq, LAST(DOWN.seqCol) AS bottom_seq," + + " LAST(UP.seqCol) AS end_seq, LAST(DOWN.priceCol) AS bottom_price" + + " ONE ROW PER MATCH" + + " AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (STRT DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr ORDER BY symbolCol, start_seq"; + // STRT consumes one row, so the second AAPL V (seq 5..7) is unreachable: the first match ends at seq 5 and + // SKIP PAST LAST ROW resumes at seq 6, leaving no row for STRT before the fall at seq 6. + assertMatchRows(query, new Object[][]{ + {"AAPL", 1, 1, 3, 5, 5}, + {"AMZN", 1, 1, 4, 5, 5}, + {"MSFT", 1, 1, 2, 3, 3} + }); + } + + /// THE critical default. SQL:2016, Trino, Snowflake and Oracle all default an omitted AFTER MATCH clause to + /// SKIP PAST LAST ROW, while Calcite's `SqlToRelConverter` silently substitutes SKIP TO NEXT ROW. The two + /// differ in whether matches may overlap, so a regression here changes results without changing anything visible. + @Test(dataProvider = "useV2QueryEngine") + public void testOmittedAfterMatchIsSkipPastLastRow(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + Object[][] nonOverlapping = new Object[][]{ + {"AAPL", 2, 8, 12}, + {"AAPL", 6, 7, 11}, + {"AMZN", 2, 15, 25}, + {"MSFT", 2, 3, 8} + }; + assertMatchRows(vShapeQuery("AFTER MATCH SKIP PAST LAST ROW"), nonOverlapping); + assertMatchRows(vShapeQuery(""), nonOverlapping); + + // Same rows plus the overlapping ones. An engine that kept Calcite's default would return exactly this for the + // two queries above. + assertMatchRows(vShapeQuery("AFTER MATCH SKIP TO NEXT ROW"), new Object[][]{ + {"AAPL", 2, 8, 12}, + {"AAPL", 3, 5, 12}, + {"AAPL", 6, 7, 11}, + {"AMZN", 2, 15, 25}, + {"AMZN", 3, 10, 25}, + {"AMZN", 4, 5, 25}, + {"MSFT", 2, 3, 8} + }); + } + + /// All four skip modes over the same pattern, each producing a different row set. `S{2}` pushes the first row + /// mapped to `U` two rows past the start of the match, which is what makes SKIP TO FIRST U differ from + /// SKIP TO NEXT ROW. + @Test(dataProvider = "useV2QueryEngine") + public void testSkipToFirstAndLastOfPatternVariable(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + // NFLX (8 strictly increasing rows) admits a match at every start row 1..5; each mode consumes them differently. + assertMatchRows(skipTargetQuery("AFTER MATCH SKIP PAST LAST ROW"), new Object[][]{ + {"AAPL", 2, 5}, + {"NFLX", 1, 4}, + {"NFLX", 5, 8} + }); + assertMatchRows(skipTargetQuery("AFTER MATCH SKIP TO NEXT ROW"), new Object[][]{ + {"AAPL", 2, 5}, + {"NFLX", 1, 4}, + {"NFLX", 2, 5}, + {"NFLX", 3, 6}, + {"NFLX", 4, 7}, + {"NFLX", 5, 8} + }); + // FIRST(U) is the third row of the match, so matching resumes two rows in. + assertMatchRows(skipTargetQuery("AFTER MATCH SKIP TO FIRST U"), new Object[][]{ + {"AAPL", 2, 5}, + {"NFLX", 1, 4}, + {"NFLX", 3, 6}, + {"NFLX", 5, 8} + }); + // LAST(U) is the fourth and last row of the match, so matching resumes three rows in. + assertMatchRows(skipTargetQuery("AFTER MATCH SKIP TO LAST U"), new Object[][]{ + {"AAPL", 2, 5}, + {"NFLX", 1, 4}, + {"NFLX", 4, 7} + }); + } + + /// A greedy quantifier takes the longest run it can, a reluctant one the shortest. On NFLX the difference is one + /// seven-row match versus four two-row matches. + @Test(dataProvider = "useV2QueryEngine") + public void testGreedyVersusReluctantQuantifier(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + assertMatchRows(quantifierQuery("+"), new Object[][]{ + {"AAPL", 3, 2, 5}, + {"AAPL", 6, 1, 7}, + {"AMZN", 4, 1, 5}, + {"MSFT", 2, 1, 3}, + {"NFLX", 1, 7, 8} + }); + assertMatchRows(quantifierQuery("+?"), new Object[][]{ + {"AAPL", 3, 1, 4}, + {"AAPL", 6, 1, 7}, + {"AMZN", 4, 1, 5}, + {"MSFT", 2, 1, 3}, + {"NFLX", 1, 1, 2}, + {"NFLX", 3, 1, 4}, + {"NFLX", 5, 1, 6}, + {"NFLX", 7, 1, 8} + }); + } + + /// Alternation prefers the leftmost branch that lets the whole pattern complete, which CLASSIFIER() reports. + @Test(dataProvider = "useV2QueryEngine") + public void testAlternation(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(S.seqCol) AS start_seq, CLASSIFIER() AS cls" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (S (UP | DN))" + + " DEFINE UP AS UP.priceCol > PREV(UP.priceCol), DN AS DN.priceCol < PREV(DN.priceCol)" + + ") AS mr ORDER BY symbolCol, start_seq"; + // GOOG is flat, so neither branch can ever match and the partition contributes nothing. + assertMatchRows(query, new Object[][]{ + {"AAPL", 1, "DN"}, + {"AAPL", 3, "UP"}, + {"AAPL", 5, "DN"}, + {"AMZN", 1, "DN"}, + {"AMZN", 3, "DN"}, + {"MSFT", 1, "DN"}, + {"NFLX", 1, "UP"}, + {"NFLX", 3, "UP"}, + {"NFLX", 5, "UP"}, + {"NFLX", 7, "UP"} + }); + } + + /// A bounded quantifier honours both bounds: AAPL can only supply the minimum of two rows, NFLX could supply seven + /// but is capped at the maximum of three. + @Test(dataProvider = "useV2QueryEngine") + public void testBoundedQuantifier(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(S.seqCol) AS start_seq, COUNT(UP.priceCol) AS up_count, LAST(UP.seqCol) AS end_seq" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (S UP{2,3})" + + " DEFINE UP AS UP.priceCol > PREV(UP.priceCol)" + + ") AS mr ORDER BY symbolCol, start_seq"; + assertMatchRows(query, new Object[][]{ + {"AAPL", 3, 2, 5}, + {"NFLX", 1, 3, 4}, + {"NFLX", 5, 3, 8} + }); + } + + /// MATCH_NUMBER() restarts at 1 in every partition, CLASSIFIER() reports the label of the final row of the match, + /// and each single-variable aggregate sees only the rows bound to its own pattern variable. + /// + /// COUNT is pinned here because `MatchTerm.Aggregate` used to route `COUNT()` through the + /// accumulator, which has no COUNT branch. AVG is pinned because an integral result type would truncate 6.5 to 6. + @Test(dataProvider = "useV2QueryEngine") + public void testMatchNumberClassifierAndAggregates(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES MATCH_NUMBER() AS mno, CLASSIFIER() AS cls, COUNT(DOWN.priceCol) AS down_count," + + " COUNT(*) AS all_count, SUM(DOWN.priceCol) AS down_sum, MIN(DOWN.priceCol) AS down_min," + + " MAX(DOWN.priceCol) AS down_max, AVG(DOWN.priceCol) AS down_avg" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr ORDER BY symbolCol, mno"; + assertMatchRows(query, new Object[][]{ + // AAPL match 1 maps prices 8 and 5 to DOWN, prices 9 and 12 to UP. + {"AAPL", 1, "UP", 2, 4, 13, 5, 8, 6.5d}, + {"AAPL", 2, "UP", 1, 2, 7, 7, 7, 7.0d}, + {"AMZN", 1, "UP", 3, 4, 30, 5, 15, 10.0d}, + {"MSFT", 1, "UP", 1, 2, 3, 3, 3, 3.0d} + }); + } + + /// PREV and NEXT are bounded by the partition, not by the input: at a partition boundary they must yield NULL rather + /// than the neighbouring partition's row. + /// + /// The navigations live in DEFINE rather than in MEASURES because Calcite's `SqlValidatorImpl + /// .PatternValidator` unconditionally rejects PREV/NEXT inside a MEASURES item (see + /// [#testDeferredConstructsAreRejected]), so DEFINE is the only place a query can reach them. + /// + /// `PREV(...)` being NULL makes the whole predicate NULL, which SQL:2016 treats as "not matched". So the + /// proof that PREV stops at the partition start is that **no** row with `seqCol = 1` appears in the first + /// result, and the proof that NEXT stops at the partition end is that no partition's last row appears in the second + /// one - even though every one of those rows would satisfy the predicate against a neighbouring partition's price. + @Test(dataProvider = "useV2QueryEngine") + public void testPrevAndNextAreBoundedByPartition(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + // Rows whose predecessor within the partition is cheaper. seqCol = 1 is absent for every symbol. + assertMatchRows(neighbourQuery("PREV(A.priceCol) < A.priceCol"), new Object[][]{ + {"AAPL", 4, 9}, {"AAPL", 5, 12}, {"AAPL", 7, 11}, + {"AMZN", 5, 25}, + {"MSFT", 3, 8}, + {"NFLX", 2, 2}, {"NFLX", 3, 3}, {"NFLX", 4, 4}, {"NFLX", 5, 5}, {"NFLX", 6, 6}, {"NFLX", 7, 7}, + {"NFLX", 8, 8} + }); + // Rows whose successor within the partition is dearer. The last row of AAPL (7), AMZN (5), GOOG (3), MSFT (3) + // and NFLX (8) is absent from every one of them. + assertMatchRows(neighbourQuery("NEXT(A.priceCol) > A.priceCol"), new Object[][]{ + {"AAPL", 3, 5}, {"AAPL", 4, 9}, {"AAPL", 6, 7}, + {"AMZN", 4, 5}, + {"MSFT", 2, 3}, + {"NFLX", 1, 1}, {"NFLX", 2, 2}, {"NFLX", 3, 3}, {"NFLX", 4, 4}, {"NFLX", 5, 5}, {"NFLX", 6, 6}, + {"NFLX", 7, 7} + }); + + // And directly: the NULL is observable, and it happens exactly once per partition on each side. + assertMatchRows(neighbourQuery("PREV(A.priceCol) IS NULL"), new Object[][]{ + {"AAPL", 1, 10}, {"AMZN", 1, 20}, {"GOOG", 1, 4}, {"MSFT", 1, 5}, {"NFLX", 1, 1} + }); + assertMatchRows(neighbourQuery("NEXT(A.priceCol) IS NULL"), new Object[][]{ + {"AAPL", 7, 11}, {"AMZN", 5, 25}, {"GOOG", 3, 4}, {"MSFT", 3, 8}, {"NFLX", 8, 8} + }); + } + + /// Source NULLs retain SQL semantics when null handling is enabled. With null handling disabled Pinot exposes the + /// INT dimension default instead, so DEFINE and aggregates intentionally produce different results. + @Test(dataProvider = "useV2QueryEngine") + public void testStoredNullSemantics(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String nullDefinitionQuery = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES COUNT(*) AS null_count" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A+)" + + " DEFINE A AS A.nullablePriceCol IS NULL" + + ") AS mr ORDER BY symbolCol"; + assertMatchRows("SET enableNullHandling=true; " + nullDefinitionQuery, new Object[][]{ + {"GOOG", 3}, {"MSFT", 1} + }); + assertMatchRows("SET enableNullHandling=false; " + nullDefinitionQuery, new Object[][]{}); + + String nullAggregateQuery = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES COUNT(A.nullablePriceCol) AS value_count, COUNT(*) AS all_count," + + " SUM(A.nullablePriceCol) AS value_sum" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A+)" + + " DEFINE A AS A.seqCol > 0" + + ") AS mr ORDER BY symbolCol"; + assertMatchRows("SET enableNullHandling=true; " + nullAggregateQuery, new Object[][]{ + {"AAPL", 7, 7, 62}, {"AMZN", 5, 5, 75}, {"GOOG", 0, 3, null}, {"MSFT", 2, 3, 13}, + {"NFLX", 8, 8, 36} + }); + assertMatchRows("SET enableNullHandling=false; " + nullAggregateQuery, new Object[][]{ + {"AAPL", 7, 7, 62}, {"AMZN", 5, 5, 75}, + {"GOOG", 3, 3, 3L * FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT}, + {"MSFT", 3, 3, 13L + FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT}, {"NFLX", 8, 8, 36} + }); + } + + /// PARTITION BY genuinely isolates matches. The `^` and `$` anchors are relative to the partition, so + /// each must match exactly once per symbol; if partitioning leaked they would match once for the whole table. + @Test(dataProvider = "useV2QueryEngine") + public void testPartitionIsolationAcrossSegments(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + // The fixture really is spread over several segments, so the exchange below the match operator is exercised. + int numSegments = getSharedHelixResourceManager() + .getSegmentsFor(TableNameBuilder.OFFLINE.tableNameWithType(getTableName()), false).size(); + assertTrue(numSegments > 1, "Expected the fixture to span more than one segment, but found " + numSegments); + + assertMatchRows(anchoredQuery("^ A"), new Object[][]{ + {"AAPL", 1, 10}, + {"AMZN", 1, 20}, + {"GOOG", 1, 4}, + {"MSFT", 1, 5}, + {"NFLX", 1, 1} + }); + assertMatchRows(anchoredQuery("A $"), new Object[][]{ + {"AAPL", 7, 11}, + {"AMZN", 5, 25}, + {"GOOG", 3, 4}, + {"MSFT", 3, 8}, + {"NFLX", 8, 8} + }); + + // A pattern that pairs up adjacent rows. Partition sizes are 7, 5, 3, 3 and 8, so the odd row at the end of AAPL, + // GOOG, MSFT and AMZN is dropped rather than paired with the next partition's first row: 11 matches, not the 13 + // that a single 26-row stream would produce. + String pairs = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(A.seqCol) AS start_seq, LAST(B.seqCol) AS end_seq" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A B)" + + " DEFINE A AS A.priceCol > 0, B AS B.priceCol > 0" + + ") AS mr ORDER BY symbolCol, start_seq"; + assertMatchRows(pairs, new Object[][]{ + {"AAPL", 1, 2}, {"AAPL", 3, 4}, {"AAPL", 5, 6}, + {"AMZN", 1, 2}, {"AMZN", 3, 4}, + {"GOOG", 1, 2}, + {"MSFT", 1, 2}, + {"NFLX", 1, 2}, {"NFLX", 3, 4}, {"NFLX", 5, 6}, {"NFLX", 7, 8} + }); + } + + /// A multi-column PARTITION BY whose source order is not ascending input-column order. + /// + /// The Pinot schema is a `TreeMap`, so the partition columns are `priceCol(1)` and `symbolCol(3)` + /// and `PARTITION BY symbolCol, priceCol` is therefore in **descending** index order. Calcite's + /// `Match#getPartitionKeys()` is an `ImmutableBitSet`, which loses that order, while the output row type + /// keeps it - so an engine that read the partition keys off the bit set would write `priceCol`'s Integer into + /// the STRING slot and fail the whole query with a ClassCastException. + /// + /// GOOG is the load-bearing row: its three rows all cost 4, so they form one three-row partition and + /// `COUNT(*)` reports 3. Every other (symbol, price) pair is unique and yields a one-row partition. + @Test(dataProvider = "useV2QueryEngine") + public void testPartitionByKeepsItsSourceOrder(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol, priceCol" + + " ORDER BY seqCol" + + " MEASURES COUNT(*) AS cnt" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A+)" + + " DEFINE A AS A.priceCol > 0" + + ") AS mr ORDER BY symbolCol, priceCol"; + assertMatchRows(query, new Object[][]{ + {"AAPL", 5, 1}, {"AAPL", 7, 1}, {"AAPL", 8, 1}, {"AAPL", 9, 1}, {"AAPL", 10, 1}, {"AAPL", 11, 1}, + {"AAPL", 12, 1}, + {"AMZN", 5, 1}, {"AMZN", 10, 1}, {"AMZN", 15, 1}, {"AMZN", 20, 1}, {"AMZN", 25, 1}, + {"GOOG", 4, 3}, + {"MSFT", 3, 1}, {"MSFT", 5, 1}, {"MSFT", 8, 1}, + {"NFLX", 1, 1}, {"NFLX", 2, 1}, {"NFLX", 3, 1}, {"NFLX", 4, 1}, {"NFLX", 5, 1}, {"NFLX", 6, 1}, + {"NFLX", 7, 1}, {"NFLX", 8, 1} + }); + } + + /// Opting into a missing PARTITION BY executes one globally ordered match, not one partial match per worker. + @Test(dataProvider = "useV2QueryEngine") + public void testWithoutPartitionByExecutesGlobally(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + String query = "SET allowMatchRecognizeWithoutPartitionBy=true; SELECT * FROM " + getTableName() + + " MATCH_RECOGNIZE (" + + " ORDER BY symbolCol, seqCol" + + " MEASURES COUNT(*) AS row_count, FIRST(A.symbolCol) AS first_symbol," + + " LAST(A.symbolCol) AS last_symbol" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A+)" + + " DEFINE A AS A.seqCol > 0" + + ") AS mr"; + JsonNode response = assertMatchRows(query, new Object[][]{ + {getCountStarResult(), "AAPL", "NFLX"} + }); + assertEquals(response.get("numServersQueried").asInt(), 2, + "The global MATCH_RECOGNIZE regression must execute across both server workers"); + } + + /// Every deferred construct is rejected during planning with a message that names it, rather than producing a wrong + /// result or an internal Calcite failure. + @Test(dataProvider = "useV2QueryEngine") + public void testDeferredConstructsAreRejected(boolean useMultiStageQueryEngine) + throws Exception { + setUseMultiStageQueryEngine(useMultiStageQueryEngine); + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES CLASSIFIER() AS cls" + + " ALL ROWS PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr", "ALL ROWS PER MATCH is not supported yet"); + + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(DOWN.priceCol) AS p" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + " SUBSET BOTH = (DOWN, UP)" + + V_SHAPE_DEFINE + + ") AS mr", "SUBSET is not supported yet"); + + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol" + + " MEASURES LAST(DOWN.priceCol) AS p" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr", "MATCH_RECOGNIZE requires an ORDER BY clause"); + + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(DOWN.priceCol) AS p" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + " DEFINE DOWN AS COUNT(DOWN.priceCol) < 3, UP AS UP.priceCol > PREV(UP.priceCol)" + + ") AS mr", "is not supported yet in the MATCH_RECOGNIZE DEFINE clause"); + + // PERMUTE is a non-reserved keyword, so the single argument form parses as a concatenation of an undefined + // pattern variable named PERMUTE and a group. MatchRecognizeValidator turns that silently wrong plan into an + // explicit error. + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(DOWN.priceCol) AS p" + + " ONE ROW PER MATCH" + + " PATTERN (PERMUTE(DOWN))" + + " DEFINE DOWN AS DOWN.priceCol < PREV(DOWN.priceCol)" + + ") AS mr", "PERMUTE is not supported yet"); + + // Calcite's SqlValidatorImpl.PatternValidator rejects PREV/NEXT anywhere inside a MEASURES item, however they are + // nested. SQL:2016, Oracle and Trino all allow them there, and MatchTerm.Navigation implements them, but no query + // can reach that path: physical navigation is only usable from DEFINE. Pinned here so the day the restriction is + // lifted or wrapped in a Pinot specific message, this test says so. + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES PREV(A.priceCol) AS prev_price" + + " ONE ROW PER MATCH" + + " PATTERN (A)" + + " DEFINE A AS A.priceCol > 0" + + ") AS mr", "Cannot use PREV/NEXT in MEASURE"); + + // MEASURES is optional in SQL:2016, but Calcite then makes the MATCH_RECOGNIZE row type the whole *input* row + // type instead of the ONE ROW PER MATCH shape, so the query used to plan and then die on the server. + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr", "MATCH_RECOGNIZE requires a MEASURES clause"); + + // Calcite adds a measure to the row type only when its alias is still free, so a measure aliased to a PARTITION + // BY column silently disappears from the output. + assertPlanningError("SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(DOWN.priceCol) AS symbolCol" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr", "collides with a PARTITION BY column or an earlier measure alias"); + + // Calcite qualifies an unqualified column reference with the row source alias, so a pattern variable of the same + // name steals every unqualified reference from the SQL:2016 universal row pattern variable - silently changing + // both measure values and which rows match. + assertPlanningError("SELECT * FROM " + getTableName() + " AS DOWN MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(priceCol) AS p" + + " ONE ROW PER MATCH" + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ")", "collides with the row source alias"); + } + + private String neighbourQuery(String definition) { + return "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(A.seqCol) AS seq, LAST(A.priceCol) AS cur_price" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (A)" + + " DEFINE A AS " + definition + + ") AS mr ORDER BY symbolCol, seq"; + } + + private String anchoredQuery(String pattern) { + return "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES LAST(A.seqCol) AS seq, LAST(A.priceCol) AS cur_price" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (" + pattern + ")" + + " DEFINE A AS A.priceCol > 0" + + ") AS mr ORDER BY symbolCol"; + } + + private String vShapeQuery(String afterMatch) { + return "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(DOWN.seqCol) AS start_seq, FIRST(DOWN.priceCol) AS start_price," + + " LAST(UP.priceCol) AS end_price" + + " ONE ROW PER MATCH " + afterMatch + + " PATTERN (DOWN+ UP+)" + + V_SHAPE_DEFINE + + ") AS mr ORDER BY symbolCol, start_seq"; + } + + private String skipTargetQuery(String afterMatch) { + return "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(S.seqCol) AS start_seq, LAST(U.seqCol) AS end_seq" + + " ONE ROW PER MATCH " + afterMatch + + " PATTERN (S{2} U{2})" + + " DEFINE U AS U.priceCol > PREV(U.priceCol)" + + ") AS mr ORDER BY symbolCol, start_seq"; + } + + private String quantifierQuery(String quantifier) { + return "SELECT * FROM " + getTableName() + " MATCH_RECOGNIZE (" + + " PARTITION BY symbolCol ORDER BY seqCol" + + " MEASURES FIRST(S.seqCol) AS start_seq, COUNT(U.priceCol) AS u_count, LAST(U.seqCol) AS end_seq" + + " ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW" + + " PATTERN (S U" + quantifier + ")" + + " DEFINE U AS U.priceCol > PREV(U.priceCol)" + + ") AS mr ORDER BY symbolCol, start_seq"; + } + + /// Asserts the full result set of `query`, cell by cell. `null` expects a SQL NULL, a [String] + /// expects a string, a [Double] expects an approximate double and any other [Number] an exact integer. + private JsonNode assertMatchRows(String query, Object[][] expected) + throws Exception { + JsonNode response = postQuery(query); + QueryAssert.assertThat(response).hasNoExceptions(); + JsonNode rows = response.get("resultTable").get("rows"); + assertNotNull(rows, "No rows in response for query: " + query); + String context = "\nQuery: " + query + "\nRows: " + rows; + assertEquals(rows.size(), expected.length, "Unexpected number of matches." + context); + for (int i = 0; i < expected.length; i++) { + JsonNode row = rows.get(i); + Object[] expectedRow = expected[i]; + assertEquals(row.size(), expectedRow.length, "Unexpected number of columns in row " + i + "." + context); + for (int j = 0; j < expectedRow.length; j++) { + Object expectedValue = expectedRow[j]; + JsonNode actual = row.get(j); + String where = "Mismatch at row " + i + ", column " + j + "." + context; + if (expectedValue == null) { + assertTrue(actual.isNull(), where + "\nExpected NULL but got: " + actual); + } else if (expectedValue instanceof String) { + assertEquals(actual.asText(), expectedValue, where); + } else if (expectedValue instanceof Double) { + assertEquals(actual.asDouble(), (double) (Double) expectedValue, 1e-9, where); + } else { + assertEquals(actual.asLong(), ((Number) expectedValue).longValue(), where); + } + } + } + return response; + } + + private void assertPlanningError(String query, String expectedMessage) + throws Exception { + QueryAssert.assertThat(postQuery(query)).firstException().containsMessage(expectedMessage); + } + + @Override + public String getTableName() { + return DEFAULT_TABLE_NAME; + } + + @Override + public Schema createSchema() { + return new Schema.SchemaBuilder().setSchemaName(getTableName()) + .setEnableColumnBasedNullHandling(true) + .addSingleValueDimension(SYMBOL_COLUMN, FieldSpec.DataType.STRING) + .addSingleValueDimension(SEQ_COLUMN, FieldSpec.DataType.INT) + .addSingleValueDimension(PRICE_COLUMN, FieldSpec.DataType.INT) + .addSingleValueDimension(NULLABLE_PRICE_COLUMN, FieldSpec.DataType.INT) + .build(); + } + + @Override + public int getNumAvroFiles() { + return 3; + } + + @Override + protected long getCountStarResult() { + int total = 0; + for (int[] prices : PRICES) { + total += prices.length; + } + return total; + } + + @Override + public List createAvroFiles() + throws Exception { + org.apache.avro.Schema avroSchema = org.apache.avro.Schema.createRecord("myRecord", null, null, false); + org.apache.avro.Schema nullableIntSchema = org.apache.avro.Schema.createUnion(List.of( + org.apache.avro.Schema.create(org.apache.avro.Schema.Type.NULL), + org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT))); + avroSchema.setFields(List.of( + new org.apache.avro.Schema.Field(SYMBOL_COLUMN, + org.apache.avro.Schema.create(org.apache.avro.Schema.Type.STRING), null, null), + new org.apache.avro.Schema.Field(SEQ_COLUMN, + org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null), + new org.apache.avro.Schema.Field(PRICE_COLUMN, + org.apache.avro.Schema.create(org.apache.avro.Schema.Type.INT), null, null), + new org.apache.avro.Schema.Field(NULLABLE_PRICE_COLUMN, nullableIntSchema, null, null) + )); + + try (AvroFilesAndWriters avroFilesAndWriters = createAvroFilesAndWriters(avroSchema)) { + List> writers = avroFilesAndWriters.getWriters(); + // Round-robin so that consecutive rows of a partition land in different segments: the match operator must rely + // on the sort exchange below it, never on the physical layout. + int rowIndex = 0; + for (int s = 0; s < SYMBOLS.length; s++) { + int[] prices = PRICES[s]; + for (int i = 0; i < prices.length; i++) { + GenericData.Record record = new GenericData.Record(avroSchema); + record.put(SYMBOL_COLUMN, SYMBOLS[s]); + record.put(SEQ_COLUMN, i + 1); + record.put(PRICE_COLUMN, prices[i]); + boolean nullablePriceIsNull = SYMBOLS[s].equals("GOOG") || SYMBOLS[s].equals("MSFT") && i == 1; + record.put(NULLABLE_PRICE_COLUMN, nullablePriceIsNull ? null : prices[i]); + writers.get(rowIndex++ % getNumAvroFiles()).append(record); + } + } + return avroFilesAndWriters.getAvroFiles(); + } + } +} diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRule.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRule.java new file mode 100644 index 000000000000..a83f75a9baa1 --- /dev/null +++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRule.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.calcite.rel.rules; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.calcite.plan.Context; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelDistributions; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Exchange; +import org.apache.calcite.rel.core.Match; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange; +import org.apache.pinot.calcite.rel.logical.PinotLogicalSortExchange; +import org.apache.pinot.query.context.PlannerContext; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.exception.QueryException; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; + + +/// Special rule for Pinot, this rule is fixed to always insert an exchange below the MATCH_RECOGNIZE node, mirroring +/// what [PinotWindowExchangeNodeInsertRule] does for `WINDOW`. +/// +/// Row pattern recognition is only correct when every row of a `PARTITION BY` partition is evaluated by the same +/// worker, in `ORDER BY` order. The rule therefore inserts a [PinotLogicalSortExchange] hash-distributed on the +/// `PARTITION BY` keys, sorted on the receiver side. +/// +/// **The receiver-side collation is `(partitionKeys..., orderByKeys...)`, not just the `ORDER BY` keys.** Prepending +/// the partition keys makes rows arrive clustered by partition, so the runtime operator can match and flush one +/// partition at a time and release its buffer at each partition boundary. Sorting on the `ORDER BY` keys alone would +/// interleave partitions and force the operator to buffer a `Map>` for the whole input until +/// end-of-stream. An `ORDER BY` key that is also a partition key is dropped from the tail: it is constant inside a +/// partition, so re-sorting on it there is a no-op. +/// +/// Sorting happens on the receiver only. Sender-side sorting is not implemented for exchanges yet (see the same +/// note in [PinotWindowExchangeNodeInsertRule]). +/// +/// A query with no `PARTITION BY` is rejected, because hashing on zero keys sends the entire table to one worker. +/// Set the [QueryOptionKey#ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY] query option to opt into that plan. +/// +/// TODO(#19395): Support sender-side sorting so the receiver can k-way merge already sorted streams instead of +/// re-sorting. +public class PinotMatchExchangeNodeInsertRule extends RelOptRule { + public static final PinotMatchExchangeNodeInsertRule INSTANCE = + new PinotMatchExchangeNodeInsertRule(PinotRuleUtils.PINOT_REL_FACTORY); + + public PinotMatchExchangeNodeInsertRule(RelBuilderFactory factory) { + super(operand(Match.class, any()), factory, null); + } + + @Override + public boolean matches(RelOptRuleCall call) { + Match match = call.rel(0); + return !PinotRuleUtils.isExchange(match.getInput()); + } + + @Override + public void onMatch(RelOptRuleCall call) { + Match match = call.rel(0); + List partitionKeys = match.getPartitionKeys().toList(); + if (partitionKeys.isEmpty() && !isMatchWithoutPartitionByAllowed(call)) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE without a PARTITION BY clause is not allowed because the whole table has to be routed to " + + "a single worker, which does not scale and can exhaust that worker's memory. Add a PARTITION BY " + + "clause on a column with enough distinct values, or set the query option '" + + QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY + "=true' to accept single-worker " + + "execution."); + } + + RelNode input = match.getInput(); + RelCollation collation = clusterByPartitionKeys(partitionKeys, match.getOrderKeys()); + Exchange exchange; + if (collation.getFieldCollations().isEmpty()) { + // Neither PARTITION BY nor ORDER BY: nothing to sort on, so a plain exchange onto a single worker is enough. + exchange = PinotLogicalExchange.create(input, RelDistributions.hash(partitionKeys)); + } else { + exchange = PinotLogicalSortExchange.create(input, RelDistributions.hash(partitionKeys), collation, false, true); + } + call.transformTo(match.copy(match.getTraitSet(), List.of(exchange))); + } + + /// Builds the receiver-side collation `(partitionKeys..., orderByKeys...)` so that rows arrive grouped by + /// partition and, within a partition, in `ORDER BY` order. + private static RelCollation clusterByPartitionKeys(List partitionKeys, RelCollation orderKeys) { + if (partitionKeys.isEmpty()) { + return orderKeys; + } + List fieldCollations = new ArrayList<>(partitionKeys.size()); + for (int partitionKey : partitionKeys) { + // The direction is irrelevant for clustering, any total order groups the partitions together. + fieldCollations.add(new RelFieldCollation(partitionKey)); + } + Set partitionKeySet = new HashSet<>(partitionKeys); + for (RelFieldCollation fieldCollation : orderKeys.getFieldCollations()) { + // A partition key is constant within its partition, so sorting on it again is a no-op. + if (!partitionKeySet.contains(fieldCollation.getFieldIndex())) { + fieldCollations.add(fieldCollation); + } + } + return RelCollations.of(fieldCollations); + } + + private static boolean isMatchWithoutPartitionByAllowed(RelOptRuleCall call) { + PlannerContext plannerContext = getPlannerContext(call); + return plannerContext != null && Boolean.parseBoolean( + plannerContext.getOptions().get(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY)); + } + + @Nullable + private static PlannerContext getPlannerContext(RelOptRuleCall call) { + Context context = call.getPlanner().getContext(); + return context != null ? context.unwrap(PlannerContext.class) : null; + } +} diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotQueryRuleSets.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotQueryRuleSets.java index 38822d4dfad7..d73e741ee0c1 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotQueryRuleSets.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotQueryRuleSets.java @@ -278,6 +278,7 @@ private PinotQueryRuleSets() { PinotAggregateExchangeNodeInsertRule.WithoutSort.INSTANCE, PinotWindowSplitRule.INSTANCE, PinotWindowExchangeNodeInsertRule.INSTANCE, + PinotMatchExchangeNodeInsertRule.INSTANCE, PinotSetOpExchangeNodeInsertRule.INSTANCE, // apply dynamic broadcast rule after exchange is inserted diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java index e05ee827547d..f3f5a71a087a 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotOperatorTable.java @@ -185,6 +185,10 @@ public static PinotOperatorTable instance(boolean nullHandlingEnabled) { SqlStdOperatorTable.NOT, SqlStdOperatorTable.UNARY_MINUS, SqlStdOperatorTable.UNARY_PLUS, + /// MATCH_RECOGNIZE running / final semantics modifiers, e.g. `FINAL LAST(A.price)`. These are only legal inside + /// a MATCH_RECOGNIZE MEASURES / DEFINE clause; Calcite rejects them anywhere else. + SqlStdOperatorTable.FINAL, + SqlStdOperatorTable.RUNNING, // AGGREGATE OPERATORS SqlStdOperatorTable.COUNT, @@ -261,6 +265,15 @@ public static PinotOperatorTable instance(boolean nullHandlingEnabled) { SqlStdOperatorTable.SIN, SqlStdOperatorTable.TAN, SqlStdOperatorTable.TRUNCATE, + /// MATCH_RECOGNIZE (row pattern recognition) navigation and classification functions. The parser already binds + /// these calls to the standard operators, but they must also be visible in this table so that the validator can + /// resolve their overloads while deriving types. + SqlStdOperatorTable.FIRST, + SqlStdOperatorTable.LAST, + SqlStdOperatorTable.PREV, + SqlStdOperatorTable.NEXT, + SqlStdOperatorTable.CLASSIFIER, + SqlStdOperatorTable.MATCH_NUMBER, SqlStdOperatorTable.FLOOR, SqlStdOperatorTable.CEIL, SqlStdOperatorTable.TIMESTAMP_ADD, diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java index bfa24cac481e..0d759243d236 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/QueryEnvironment.java @@ -93,6 +93,7 @@ import org.apache.pinot.query.routing.WorkerManager; import org.apache.pinot.query.type.TypeFactory; import org.apache.pinot.query.validate.BytesCastVisitor; +import org.apache.pinot.query.validate.MatchRecognizeValidator; import org.apache.pinot.query.validate.RowExpressionValidationVisitor; import org.apache.pinot.spi.exception.QueryErrorCode; import org.apache.pinot.spi.exception.QueryException; @@ -419,6 +420,19 @@ private RelRoot compileQuery(SqlNode sqlNode, PlannerContext plannerContext) { /// in case it was legal to apply an automatic cast for these types!). private SqlNode validate(SqlNode sqlNode, PlannerContext plannerContext) { try { + // MATCH_RECOGNIZE checks and rewrites must run on the raw SqlNode tree: the omitted-vs-explicit AFTER MATCH + // distinction is lost during conversion to RelNode, and some unsupported constructs fail inside + // SqlToRelConverter with an assertion instead of a usable message. See MatchRecognizeValidator. + MatchRecognizeValidator matchRecognizeValidator = new MatchRecognizeValidator(); + sqlNode.accept(matchRecognizeValidator); + if (matchRecognizeValidator.hasMatchRecognize() && plannerContext.isUsePhysicalOptimizer()) { + PhysicalPlannerContext physicalPlannerContext = plannerContext.getPhysicalPlannerContext(); + boolean useLiteMode = physicalPlannerContext != null && physicalPlannerContext.isUseLiteMode(); + throw new MatchRecognizeValidator.UnsupportedMatchRecognizeException( + "MATCH_RECOGNIZE is not supported by the multi-stage physical optimizer" + + (useLiteMode ? " in lite mode" : "") + + ". Retry with the query option 'usePhysicalOptimizer=false'."); + } SqlNode validated = plannerContext.getValidator().validate(sqlNode); if (!validated.getKind().belongsTo(SqlKind.QUERY)) { throw new IllegalArgumentException("Unsupported SQL query, failed to validate query:\n" + sqlNode); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/ExplainNodeSimplifier.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/ExplainNodeSimplifier.java index c98b29532e05..458b355fdd5e 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/ExplainNodeSimplifier.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/ExplainNodeSimplifier.java @@ -33,6 +33,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -169,6 +170,11 @@ public PlanNode visitWindow(WindowNode node, Void context) { return defaultNode(node); } + @Override + public PlanNode visitMatch(MatchNode node, Void context) { + return defaultNode(node); + } + @Override public PlanNode visitSetOp(SetOpNode node, Void context) { return defaultNode(node); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PhysicalExplainPlanVisitor.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PhysicalExplainPlanVisitor.java index 9759f0e2a89f..6a3d3cc24dfa 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PhysicalExplainPlanVisitor.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PhysicalExplainPlanVisitor.java @@ -36,6 +36,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -134,6 +135,14 @@ public StringBuilder visitWindow(WindowNode node, Context context) { return visitSimpleNode(node, context); } + @Override + public StringBuilder visitMatch(MatchNode node, Context context) { + // MatchNode#explain() is just "MATCH_RECOGNIZE", which would render two different patterns identically. Append + // the pattern so the physical plan of a MATCH_RECOGNIZE query is self-describing. + appendInfo(node, context).append("(pattern=[").append(node.getPatternString()).append("])").append('\n'); + return node.getInputs().get(0).visit(this, context.next(false, context._host, context._workerId)); + } + @Override public StringBuilder visitSetOp(SetOpNode setOpNode, Context context) { appendInfo(setOpNode, context).append('\n'); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeMerger.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeMerger.java index 2c7635bc49b4..ffe474651797 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeMerger.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeMerger.java @@ -38,6 +38,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -443,6 +444,46 @@ public PlanNode visitWindow(WindowNode node, PlanNode context) { return node.withInputs(children); } + @Nullable + @Override + public PlanNode visitMatch(MatchNode node, PlanNode context) { + if (context.getClass() != MatchNode.class) { + return null; + } + MatchNode otherNode = (MatchNode) context; + // Every field of a MatchNode changes which rows are matched or what is emitted, so all of them must agree + // before two nodes can be considered the same plan. + if (!node.getPatternSymbols().equals(otherNode.getPatternSymbols())) { + return null; + } + if (!node.getPattern().equals(otherNode.getPattern())) { + return null; + } + if (!node.getMeasures().equals(otherNode.getMeasures())) { + return null; + } + if (!node.getPartitionKeys().equals(otherNode.getPartitionKeys())) { + return null; + } + if (!node.getCollations().equals(otherNode.getCollations())) { + return null; + } + if (node.getAfterMatchSkipMode() != otherNode.getAfterMatchSkipMode()) { + return null; + } + if (node.getAfterMatchSkipToSymbolOrdinal() != otherNode.getAfterMatchSkipToSymbolOrdinal()) { + return null; + } + if (node.getRowsPerMatchMode() != otherNode.getRowsPerMatchMode()) { + return null; + } + List children = mergeChildren(node, context); + if (children == null) { + return null; + } + return node.withInputs(children); + } + @Nullable @Override public PlanNode visitSetOp(SetOpNode setOpNode, PlanNode context) { diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeSorter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeSorter.java index 0cecbfca296d..21520a181b96 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeSorter.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/PlanNodeSorter.java @@ -32,6 +32,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -130,6 +131,11 @@ public PlanNode visitWindow(WindowNode node, Comparator comparator) { return defaultNode(node, comparator); } + @Override + public PlanNode visitMatch(MatchNode node, Comparator comparator) { + return defaultNode(node, comparator); + } + @Override public PlanNode visitSetOp(SetOpNode node, Comparator comparator) { return defaultNode(node, comparator); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/EquivalentStagesFinder.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/EquivalentStagesFinder.java index 2ae70264e5c2..95d05106d35d 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/EquivalentStagesFinder.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/EquivalentStagesFinder.java @@ -29,6 +29,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -330,6 +331,23 @@ public Boolean visitWindow(WindowNode node1, PlanNode node2) { && Objects.equals(node1.getConstants(), that.getConstants()); } + @Override + public Boolean visitMatch(MatchNode node1, PlanNode node2) { + if (!(node2 instanceof MatchNode)) { + return false; + } + MatchNode that = (MatchNode) node2; + return areBaseNodesEquivalent(node1, node2) + && Objects.equals(node1.getPatternSymbols(), that.getPatternSymbols()) + && Objects.equals(node1.getPattern(), that.getPattern()) + && Objects.equals(node1.getMeasures(), that.getMeasures()) + && Objects.equals(node1.getPartitionKeys(), that.getPartitionKeys()) + && Objects.equals(node1.getCollations(), that.getCollations()) + && node1.getAfterMatchSkipMode() == that.getAfterMatchSkipMode() + && node1.getAfterMatchSkipToSymbolOrdinal() == that.getAfterMatchSkipToSymbolOrdinal() + && node1.getRowsPerMatchMode() == that.getRowsPerMatchMode(); + } + @Override public Boolean visitSetOp(SetOpNode node1, PlanNode node2) { if (!(node2 instanceof SetOpNode)) { diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanFragmenter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanFragmenter.java index e6294263a236..09626309d0ce 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanFragmenter.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanFragmenter.java @@ -35,6 +35,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -166,6 +167,11 @@ public PlanNode visitWindow(WindowNode node, Context context) { return process(node, context); } + @Override + public PlanNode visitMatch(MatchNode node, Context context) { + return process(node, context); + } + @Override public PlanNode visitSetOp(SetOpNode node, Context context) { return process(node, context); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverter.java index fe16e9ba70d4..a92fc02b59e9 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverter.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverter.java @@ -19,8 +19,10 @@ package org.apache.pinot.query.planner.logical; import com.google.common.base.Preconditions; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -36,17 +38,23 @@ import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.core.Window; import org.apache.calcite.rel.logical.LogicalIntersect; +import org.apache.calcite.rel.logical.LogicalMatch; import org.apache.calcite.rel.logical.LogicalMinus; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.logical.LogicalWindow; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.rex.RexWindowBounds; import org.apache.calcite.rex.RexWindowExclusion; import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlBinaryOperator; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.pinot.common.proto.Plan; @@ -61,9 +69,12 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; +import org.apache.pinot.query.planner.plannode.RowPattern; import org.apache.pinot.query.planner.plannode.SetOpNode; import org.apache.pinot.query.planner.plannode.SortNode; import org.apache.pinot.query.planner.plannode.TableScanNode; @@ -393,6 +404,133 @@ public Void visitWindow(WindowNode node, Void context) { return null; } + @Override + public Void visitMatch(MatchNode node, Void context) { + List inputs = inputsAsList(node); + try { + Preconditions.checkArgument(inputs.size() == 1, "Match node should have exactly one input"); + _builder.push(toLogicalMatch(node, inputs.get(0))); + } catch (RuntimeException e) { + LOGGER.warn("Failed to convert match node: {}", node, e); + _builder.push(new PinotExplainedRelNode(_builder.getCluster(), "UnknownMatch", Map.of(), + node.getDataSchema(), inputs)); + } + return null; + } + + private LogicalMatch toLogicalMatch(MatchNode node, RelNode input) { + List symbols = node.getPatternSymbols(); + Map patternDefinitions = new LinkedHashMap<>(); + Map measures = new LinkedHashMap<>(); + // MEASURES expressions and DEFINE predicates are expressed over the input row, so the input has to be on the + // RelBuilder stack while they are converted. It is always popped again, including on failure. + _builder.push(input); + try { + for (PatternSymbol symbol : symbols) { + RexExpression definition = symbol.getDefinition(); + if (definition != null) { + patternDefinitions.put(symbol.getName(), RexExpressionUtils.toRexNode(_builder, definition)); + } + } + for (MatchNode.Measure measure : node.getMeasures()) { + measures.put(measure.getName(), RexExpressionUtils.toRexNode(_builder, measure.getExpression())); + } + } finally { + _builder.build(); + } + + // MatchNode carries the `^` / `$` anchors as tree nodes; Calcite carries them as booleans on the rel. + RowPattern pattern = node.getPattern(); + RelDataType rowType = node.getDataSchema().toRelDataType(_builder.getTypeFactory()); + return LogicalMatch.create(input, rowType, toPatternRexNode(stripAnchors(pattern), symbols), + hasAnchor(pattern, RowPattern.Kind.ANCHOR_START), hasAnchor(pattern, RowPattern.Kind.ANCHOR_END), + patternDefinitions, measures, toAfterRexNode(node, symbols), Map.of(), + node.getRowsPerMatchMode() == MatchNode.RowsPerMatchMode.ALL_ROWS_PER_MATCH, + ImmutableBitSet.of(node.getPartitionKeys()), RelCollations.of(node.getCollations()), null); + } + + private static boolean hasAnchor(RowPattern pattern, RowPattern.Kind anchor) { + if (pattern.getKind() != RowPattern.Kind.CONCAT) { + return false; + } + List children = ((RowPattern.Concat) pattern).getChildren(); + return anchor == RowPattern.Kind.ANCHOR_START ? children.get(0).getKind() == anchor + : children.get(children.size() - 1).getKind() == anchor; + } + + private static RowPattern stripAnchors(RowPattern pattern) { + if (pattern.getKind() != RowPattern.Kind.CONCAT) { + return pattern; + } + List children = ((RowPattern.Concat) pattern).getChildren(); + int from = children.get(0).getKind() == RowPattern.Kind.ANCHOR_START ? 1 : 0; + int to = children.get(children.size() - 1).getKind() == RowPattern.Kind.ANCHOR_END ? children.size() - 1 + : children.size(); + if (from == 0 && to == children.size()) { + return pattern; + } + List stripped = children.subList(from, to); + Preconditions.checkArgument(!stripped.isEmpty(), "Match node pattern contains only anchors"); + return stripped.size() == 1 ? stripped.get(0) : new RowPattern.Concat(stripped); + } + + /// Rebuilds Calcite's `RexCall` encoding of the PATTERN clause: PATTERN_CONCAT and PATTERN_ALTER are binary + /// and left associative, quantifiers carry their bounds and reluctant flag as literal operands, and a pattern + /// variable is a plain string literal. + private RexNode toPatternRexNode(RowPattern pattern, List symbols) { + RexBuilder rexBuilder = _builder.getRexBuilder(); + switch (pattern.getKind()) { + case SYMBOL: + return rexBuilder.makeLiteral(symbols.get(((RowPattern.Symbol) pattern).getSymbolOrdinal()).getName()); + case CONCAT: + return toPatternRexCall(SqlStdOperatorTable.PATTERN_CONCAT, ((RowPattern.Concat) pattern).getChildren(), + symbols); + case ALTERNATE: + return toPatternRexCall(SqlStdOperatorTable.PATTERN_ALTER, ((RowPattern.Alternate) pattern).getChildren(), + symbols); + case QUANTIFIER: + RowPattern.Quantifier quantifier = (RowPattern.Quantifier) pattern; + return rexBuilder.makeCall(_builder.getTypeFactory().createUnknownType(), + SqlStdOperatorTable.PATTERN_QUANTIFIER, + List.of(toPatternRexNode(quantifier.getChild(), symbols), + rexBuilder.makeExactLiteral(BigDecimal.valueOf(quantifier.getMinRepeat())), + rexBuilder.makeExactLiteral(BigDecimal.valueOf(quantifier.getMaxRepeat())), + rexBuilder.makeLiteral(!quantifier.isGreedy()))); + default: + throw new IllegalStateException("Unsupported row pattern kind: " + pattern.getKind()); + } + } + + private RexNode toPatternRexCall(SqlBinaryOperator operator, List children, + List symbols) { + Preconditions.checkArgument(!children.isEmpty(), "Row pattern %s has no children", operator.getName()); + RexNode result = toPatternRexNode(children.get(0), symbols); + for (int i = 1; i < children.size(); i++) { + result = _builder.getRexBuilder().makeCall(_builder.getTypeFactory().createUnknownType(), operator, + List.of(result, toPatternRexNode(children.get(i), symbols))); + } + return result; + } + + private RexNode toAfterRexNode(MatchNode node, List symbols) { + RexBuilder rexBuilder = _builder.getRexBuilder(); + switch (node.getAfterMatchSkipMode()) { + case PAST_LAST_ROW: + return rexBuilder.makeFlag(SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW); + case TO_NEXT_ROW: + return rexBuilder.makeFlag(SqlMatchRecognize.AfterOption.SKIP_TO_NEXT_ROW); + case TO_FIRST: + case TO_LAST: + SqlOperator operator = node.getAfterMatchSkipMode() == MatchNode.AfterMatchSkipMode.TO_FIRST + ? SqlMatchRecognize.SKIP_TO_FIRST : SqlMatchRecognize.SKIP_TO_LAST; + String name = symbols.get(node.getAfterMatchSkipToSymbolOrdinal()).getName(); + return rexBuilder.makeCall(_builder.getTypeFactory().createUnknownType(), operator, + List.of(rexBuilder.makeLiteral(name))); + default: + throw new IllegalStateException("Unsupported after match skip mode: " + node.getAfterMatchSkipMode()); + } + } + private static RexWindowExclusion toRexWindowExclusion(WindowNode.WindowExclusion exclude) { switch (exclude) { case NO_OTHERS: diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java index 6198f1cbcb0f..80d07afe72cb 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java @@ -22,7 +22,9 @@ import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptTable; @@ -34,6 +36,7 @@ import org.apache.calcite.rel.core.Exchange; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Match; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.TableScan; @@ -50,13 +53,18 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rel.type.RelRecordType; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexWindowExclusion; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchRecognize; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; import org.apache.pinot.calcite.rel.hint.PinotHintOptions; import org.apache.pinot.calcite.rel.logical.PinotLogicalAggregate; import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange; @@ -75,15 +83,20 @@ import org.apache.pinot.query.planner.plannode.ExchangeNode; import org.apache.pinot.query.planner.plannode.FilterNode; import org.apache.pinot.query.planner.plannode.JoinNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNode.NodeHint; import org.apache.pinot.query.planner.plannode.ProjectNode; +import org.apache.pinot.query.planner.plannode.RowPattern; import org.apache.pinot.query.planner.plannode.SetOpNode; import org.apache.pinot.query.planner.plannode.SortNode; import org.apache.pinot.query.planner.plannode.TableScanNode; import org.apache.pinot.query.planner.plannode.UnnestNode; import org.apache.pinot.query.planner.plannode.ValueNode; import org.apache.pinot.query.planner.plannode.WindowNode; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.exception.QueryException; import org.apache.pinot.spi.utils.CommonConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,6 +106,13 @@ public final class RelToPlanNodeConverter { private static final Logger LOGGER = LoggerFactory.getLogger(RelToPlanNodeConverter.class); private static final int DEFAULT_STAGE_ID = -1; + /// Largest repetition count accepted in a MATCH_RECOGNIZE pattern quantifier such as `A{n,m}`. Every + /// repetition expands into pattern matcher state, so an unbounded count would let a tiny query request an + /// arbitrary amount of work. BigQuery applies the same limit. + static final int MAX_PATTERN_QUANTIFIER_BOUND = 10_000; + /// The value Calcite's parser writes for an unspecified MATCH_RECOGNIZE quantifier bound, i.e. the maximum of + /// `*` / `+` / `{n,}` and the minimum of `{,m}`. + private static final int UNSPECIFIED_QUANTIFIER_BOUND = -1; /// Pattern used to detect Calcite/Pinot auto-generated aliases such as expr$0, item0, ord0, arr0, unnest_col_0, col0. /// The matcher is case-insensitive because connectors may emit the aliases in different cases (e.g., EXPR$0 vs /// expr$0). @@ -166,6 +186,8 @@ public PlanNode toPlanNode(RelNode node) { _windowFunctionFound = true; } result = convertLogicalWindow((LogicalWindow) node); + } else if (node instanceof Match) { + result = convertLogicalMatch((Match) node); } else if (node instanceof LogicalValues) { result = convertLogicalValues((LogicalValues) node); } else if (node instanceof SetOp) { @@ -681,6 +703,376 @@ private WindowNode convertLogicalWindow(LogicalWindow node) { aggCalls, windowFrameType, lowerBound, upperBound, fromRexWindowExclusion(windowGroup.exclude), constants); } + /// Converts Calcite's [Match] (SQL:2016 MATCH_RECOGNIZE) into a [MatchNode]. + /// + /// The main job is lowering Calcite's encoding of the `PATTERN` clause - a `RexCall` tree of + /// PATTERN_CONCAT / PATTERN_ALTER / PATTERN_QUANTIFIER over string literals - into the explicit + /// [RowPattern] tree, and binding every pattern variable reference to an ordinal in the symbol table so that + /// the operator never string-matches variable names. + /// + /// [org.apache.pinot.query.validate.MatchRecognizeValidator] runs on the `SqlNode` tree before + /// conversion: it rewrites an omitted `AFTER MATCH` to `SKIP PAST LAST ROW` (the SQL:2016 default, not + /// Calcite's `SKIP TO NEXT ROW`) and rejects the constructs that are not supported yet. No default is + /// re-applied here. The rejections repeated below are a safety net: [MatchNode] cannot represent those + /// constructs, so dropping one silently would return wrong rows. + private MatchNode convertLogicalMatch(Match node) { + if (node.isAllRows()) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the default) and " + + "expose the per-match values you need through the MEASURES clause."); + } + if (!node.getSubsets().isEmpty()) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "SUBSET is not supported yet in MATCH_RECOGNIZE. Remove the SUBSET clause and reference the individual " + + "pattern variables directly."); + } + if (node.getInterval() != null) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "WITHIN is not supported yet in MATCH_RECOGNIZE. Remove the WITHIN clause and bound the match with a " + + "predicate in the DEFINE clause instead."); + } + if (node.getMeasures().isEmpty()) { + // Calcite's SqlValidatorImpl.validateMatchRecognize sets the row type of a measures-less MATCH_RECOGNIZE to the + // whole input row type, which is not the ONE ROW PER MATCH shape at all. MatchRecognizeValidator already + // rejects this on the SqlNode tree; this is the safety net, because the output schema below would otherwise be + // silently wrong rather than merely unsupported. + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE requires a MEASURES clause. ONE ROW PER MATCH emits the PARTITION BY columns plus the " + + "MEASURES, so add at least one measure, e.g. MEASURES MATCH_NUMBER() AS mno."); + } + + validateMatchPartitionKeyTypes(node); + for (Map.Entry measure : node.getMeasures().entrySet()) { + validateMatchAggregateOperandTypes(measure.getValue(), measure.getKey()); + } + + // Order the symbol table by first appearance in PATTERN so the ordinals are deterministic and independent of the + // iteration order of Calcite's pattern definition map. + Map symbolOrdinals = new LinkedHashMap<>(); + collectPatternSymbols(node.getPattern(), symbolOrdinals); + Map patternDefinitions = node.getPatternDefinitions(); + // A DEFINE for a variable that never appears in PATTERN is dead code, but keep it in the table rather than + // dropping it silently. + for (String name : patternDefinitions.keySet()) { + symbolOrdinals.putIfAbsent(name, symbolOrdinals.size()); + } + + List patternSymbols = new ArrayList<>(symbolOrdinals.size()); + for (String name : symbolOrdinals.keySet()) { + RexNode definition = patternDefinitions.get(name); + // Per SQL:2016 a pattern variable with no DEFINE entry matches every row; that is modeled as a null definition. + patternSymbols.add(new PatternSymbol(name, + definition != null ? toBoundRexExpression(definition, symbolOrdinals) : null)); + } + + List measures = new ArrayList<>(node.getMeasures().size()); + for (Map.Entry measure : node.getMeasures().entrySet()) { + measures.add( + new MatchNode.Measure(measure.getKey(), toBoundRexExpression(measure.getValue(), symbolOrdinals))); + } + + RowPattern pattern = + withAnchors(toRowPattern(node.getPattern(), symbolOrdinals), node.isStrictStart(), node.isStrictEnd()); + + RexNode after = node.getAfter(); + MatchNode.AfterMatchSkipMode afterMatchSkipMode = toAfterMatchSkipMode(after); + // Only SKIP TO FIRST / SKIP TO LAST have a target variable; the other modes keep NO_SKIP_TO_SYMBOL, which must + // stay distinguishable from the legitimate ordinal 0. + boolean hasSkipTarget = afterMatchSkipMode == MatchNode.AfterMatchSkipMode.TO_FIRST + || afterMatchSkipMode == MatchNode.AfterMatchSkipMode.TO_LAST; + int afterMatchSkipToSymbolOrdinal = + hasSkipTarget ? toAfterMatchSkipToSymbolOrdinal((RexCall) after, symbolOrdinals) + : MatchNode.NO_SKIP_TO_SYMBOL; + + return new MatchNode(DEFAULT_STAGE_ID, toDataSchema(node.getRowType()), NodeHint.EMPTY, + convertInputs(node.getInputs()), patternSymbols, pattern, measures, partitionKeysInSourceOrder(node), + node.getOrderKeys().getFieldCollations(), afterMatchSkipMode, afterMatchSkipToSymbolOrdinal, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + } + + private static void validateMatchPartitionKeyTypes(Match node) { + List inputFields = node.getInput().getRowType().getFieldList(); + for (int partitionKey : node.getPartitionKeys()) { + RelDataTypeField field = inputFields.get(partitionKey); + if (field.getType().getSqlTypeName() == SqlTypeName.ARRAY) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE does not support multi-value or ARRAY PARTITION BY columns: '" + field.getName() + + "'. Partition by a single-value column instead."); + } + } + } + + private static void validateMatchAggregateOperandTypes(RexNode expression, String measureName) { + if (!(expression instanceof RexCall)) { + return; + } + RexCall call = (RexCall) expression; + if (call.getOperator() instanceof SqlAggFunction) { + for (RexNode operand : call.getOperands()) { + if (operand.getType().getSqlTypeName() == SqlTypeName.ARRAY) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE measure '" + measureName + "' applies aggregate '" + call.getOperator().getName() + + "' to a multi-value or ARRAY operand, which is not supported. Aggregate a single-value column " + + "instead."); + } + } + } + for (RexNode operand : call.getOperands()) { + validateMatchAggregateOperandTypes(operand, measureName); + } + } + + /// The input column indexes of the `PARTITION BY` keys, in `PARTITION BY` **source** order. + /// + /// [Match#getPartitionKeys()] is an [ImmutableBitSet], so `asList()` always returns the keys in + /// ascending index order and the order the user wrote them in is gone. The output row type built by + /// `SqlValidatorImpl.validateMatchRecognize` does keep it: its leading fields are the partition columns in + /// source order, named after the last component of each `PARTITION BY` identifier. `MatchOperator` fills + /// the output row positionally, so the two have to agree - otherwise + /// `PARTITION BY , ` reports each partition column's value under the other one's + /// name, or fails with a `ClassCastException` when the two types differ. + /// + /// The number of leading partition fields is derived from the measure count rather than from + /// `partitionKeys.cardinality()`, because `PARTITION BY col, col` contributes two output fields but only + /// one bit. The names are resolved against the partition key set only, case-sensitively first, because the schema may + /// be case-insensitive and the row type carries the identifier as it was written rather than as it was resolved. + private static List partitionKeysInSourceOrder(Match node) { + ImmutableBitSet partitionKeys = node.getPartitionKeys(); + List outputFieldNames = node.getRowType().getFieldNames(); + int numPartitionFields = outputFieldNames.size() - node.getMeasures().size(); + if (numPartitionFields <= 1) { + // Nothing to reorder, and no name to resolve: keep the cheap and always correct path. + return partitionKeys.asList(); + } + List inputFieldNames = node.getInput().getRowType().getFieldNames(); + List ordered = new ArrayList<>(numPartitionFields); + for (int i = 0; i < numPartitionFields; i++) { + ordered.add(resolvePartitionKey(outputFieldNames.get(i), partitionKeys, inputFieldNames)); + } + // A future Calcite change to how the Match row type is built must fail loudly rather than silently misalign rows. + Preconditions.checkState(ImmutableBitSet.of(ordered).equals(partitionKeys), + "MATCH_RECOGNIZE partition keys %s recovered from the output row type %s do not match Calcite's %s", ordered, + outputFieldNames, partitionKeys); + return ordered; + } + + private static int resolvePartitionKey(String name, ImmutableBitSet partitionKeys, List inputFieldNames) { + int caseInsensitiveMatch = -1; + for (int index : partitionKeys) { + String inputFieldName = inputFieldNames.get(index); + if (inputFieldName.equals(name)) { + return index; + } + if (inputFieldName.equalsIgnoreCase(name)) { + caseInsensitiveMatch = index; + } + } + Preconditions.checkState(caseInsensitiveMatch >= 0, + "MATCH_RECOGNIZE partition column '%s' does not resolve to any of the partition key columns %s of %s", name, + partitionKeys, inputFieldNames); + return caseInsensitiveMatch; + } + + /// Registers every pattern variable of `pattern` in `symbolOrdinals`, in order of first appearance. + private static void collectPatternSymbols(RexNode pattern, Map symbolOrdinals) { + if (pattern instanceof RexLiteral) { + symbolOrdinals.putIfAbsent(RexLiteral.stringValue(pattern), symbolOrdinals.size()); + return; + } + if (pattern instanceof RexCall) { + for (RexNode operand : patternChildren((RexCall) pattern)) { + collectPatternSymbols(operand, symbolOrdinals); + } + } + } + + /// The sub-pattern operands of a pattern call. All operands are sub-patterns except for a quantifier, whose last + /// three operands are the bounds and the reluctant flag rather than pattern variables. + private static List patternChildren(RexCall call) { + return call.getKind() == SqlKind.PATTERN_QUANTIFIER ? call.getOperands().subList(0, 1) : call.getOperands(); + } + + private RowPattern toRowPattern(RexNode pattern, Map symbolOrdinals) { + if (pattern instanceof RexLiteral) { + String name = RexLiteral.stringValue(pattern); + Integer ordinal = symbolOrdinals.get(name); + Preconditions.checkState(ordinal != null, "Unknown MATCH_RECOGNIZE pattern variable: %s", name); + return new RowPattern.Symbol(ordinal); + } + Preconditions.checkState(pattern instanceof RexCall, "Unsupported MATCH_RECOGNIZE PATTERN node: %s", pattern); + RexCall call = (RexCall) pattern; + switch (call.getKind()) { + case PATTERN_CONCAT: + return new RowPattern.Concat(flattenRowPatterns(call, symbolOrdinals)); + case PATTERN_ALTER: + return new RowPattern.Alternate(flattenRowPatterns(call, symbolOrdinals)); + case PATTERN_QUANTIFIER: + return toQuantifier(call, symbolOrdinals); + case PATTERN_PERMUTE: + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "PERMUTE is not supported yet in the MATCH_RECOGNIZE PATTERN clause. Expand the permutation into an " + + "explicit alternation, e.g. PATTERN ((A B) | (B A))."); + case PATTERN_EXCLUDED: + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "Pattern exclusions '{- -}' are not supported yet in the MATCH_RECOGNIZE PATTERN clause. Remove the " + + "exclusion; note that exclusions only affect ALL ROWS PER MATCH output, which is also not " + + "supported yet."); + default: + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "Unsupported MATCH_RECOGNIZE PATTERN construct: '" + call + "'."); + } + } + + /// Flattens the left-deep binary tree Calcite builds for `A B C` and `A | B | C` into the operands of a + /// single n-ary node. Both operators are associative and the operand order is preserved, so the SQL:2016 + /// leftmost-alternative-wins rule is unaffected. + private List flattenRowPatterns(RexCall call, Map symbolOrdinals) { + List children = new ArrayList<>(call.getOperands().size()); + flattenRowPatterns(call, call.getKind(), symbolOrdinals, children); + return children; + } + + private void flattenRowPatterns(RexNode pattern, SqlKind kind, Map symbolOrdinals, + List children) { + if (pattern instanceof RexCall && pattern.getKind() == kind) { + for (RexNode operand : ((RexCall) pattern).getOperands()) { + flattenRowPatterns(operand, kind, symbolOrdinals, children); + } + } else { + children.add(toRowPattern(pattern, symbolOrdinals)); + } + } + + private RowPattern toQuantifier(RexCall call, Map symbolOrdinals) { + List operands = call.getOperands(); + Preconditions.checkState(operands.size() == 4, + "Expecting 4 operands in MATCH_RECOGNIZE pattern quantifier, got: %s", operands.size()); + RowPattern child = toRowPattern(operands.get(0), symbolOrdinals); + // Calcite writes -1 for an unspecified bound: `*`, `+` and `{n,}` leave the maximum open, `{,m}` leaves the + // minimum open. SQL:2016 reads an omitted minimum as 0, while an omitted maximum stays unbounded. + int minRepeat = toQuantifierBound(operands.get(1), call); + int maxRepeat = toQuantifierBound(operands.get(2), call); + if (minRepeat == UNSPECIFIED_QUANTIFIER_BOUND) { + minRepeat = 0; + } + if (maxRepeat != RowPattern.Quantifier.UNBOUNDED && maxRepeat < minRepeat) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE pattern quantifier '" + call + "' has an upper bound (" + maxRepeat + + ") smaller than its lower bound (" + minRepeat + "), so it can never match."); + } + // The reluctant forms (`*?`, `+?`, `??`, `{n,m}?`) prefer the shortest match. + return new RowPattern.Quantifier(child, minRepeat, maxRepeat, !RexLiteral.booleanValue(operands.get(3))); + } + + private int toQuantifierBound(RexNode bound, RexCall call) { + int value = RexExpressionUtils.getValueAsInt(bound); + if (value < UNSPECIFIED_QUANTIFIER_BOUND) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE pattern quantifier '" + call + "' has a negative repetition bound: " + value + "."); + } + if (value > MAX_PATTERN_QUANTIFIER_BOUND) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE pattern quantifier bound " + value + " exceeds the maximum supported bound of " + + MAX_PATTERN_QUANTIFIER_BOUND + ": '" + call + "'. Every repetition expands into pattern matcher " + + "state, so rewrite the pattern with a smaller repetition count, or use an unbounded quantifier " + + "such as '*' or '+' together with a DEFINE predicate that bounds the match."); + } + return value; + } + + /// Turns Calcite's `strictStart` / `strictEnd` booleans back into the `^` and `$` anchor + /// nodes that [RowPattern] models explicitly. + private static RowPattern withAnchors(RowPattern pattern, boolean strictStart, boolean strictEnd) { + if (!strictStart && !strictEnd) { + return pattern; + } + List children = new ArrayList<>(); + if (strictStart) { + children.add(RowPattern.AnchorStart.INSTANCE); + } + if (pattern.getKind() == RowPattern.Kind.CONCAT) { + children.addAll(((RowPattern.Concat) pattern).getChildren()); + } else { + children.add(pattern); + } + if (strictEnd) { + children.add(RowPattern.AnchorEnd.INSTANCE); + } + return new RowPattern.Concat(children); + } + + private static MatchNode.AfterMatchSkipMode toAfterMatchSkipMode(RexNode after) { + // SKIP TO FIRST / SKIP TO LAST carry their target variable, so they arrive as a call; the other two modes are + // symbol literals. + if (after instanceof RexCall) { + SqlKind kind = after.getKind(); + if (kind == SqlKind.SKIP_TO_FIRST) { + return MatchNode.AfterMatchSkipMode.TO_FIRST; + } + if (kind == SqlKind.SKIP_TO_LAST) { + return MatchNode.AfterMatchSkipMode.TO_LAST; + } + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "Unsupported MATCH_RECOGNIZE AFTER MATCH SKIP clause: '" + after + "'."); + } + Preconditions.checkState(after instanceof RexLiteral, + "Expecting a literal for MATCH_RECOGNIZE AFTER MATCH SKIP, got: %s", after); + Object option = ((RexLiteral) after).getValue(); + if (option == SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW) { + return MatchNode.AfterMatchSkipMode.PAST_LAST_ROW; + } + if (option == SqlMatchRecognize.AfterOption.SKIP_TO_NEXT_ROW) { + return MatchNode.AfterMatchSkipMode.TO_NEXT_ROW; + } + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "Unsupported MATCH_RECOGNIZE AFTER MATCH SKIP clause: '" + after + "'."); + } + + private static int toAfterMatchSkipToSymbolOrdinal(RexCall after, Map symbolOrdinals) { + String name = RexLiteral.stringValue(after.getOperands().get(0)); + Integer ordinal = symbolOrdinals.get(name); + if (ordinal == null) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "MATCH_RECOGNIZE AFTER MATCH SKIP TO references pattern variable '" + name + + "', which does not appear in the PATTERN clause."); + } + return ordinal; + } + + private RexExpression toBoundRexExpression(RexNode rexNode, Map symbolOrdinals) { + return bindPatternFieldRefs(RexExpressionUtils.fromRexNode(rexNode), symbolOrdinals); + } + + /// Binds every [RexExpression.PatternFieldRef] in `expression` to its pattern symbol ordinal. + /// [RexExpressionUtils] has no symbol table, so the references it produces are unresolved; the plan + /// serializer rejects an unresolved reference rather than writing an ambiguous one to the wire. + /// + /// An alpha that is not a pattern variable is the SQL:2016 universal row pattern variable: Calcite has no + /// dedicated representation for an unqualified column reference such as `price` in + /// `DEFINE UP AS price > PREV(price)` and reuses the row source alias instead. Those bind to + /// [RexExpression.PatternFieldRef#UNIVERSAL_SYMBOL_ORDINAL]. + private RexExpression bindPatternFieldRefs(RexExpression expression, Map symbolOrdinals) { + if (expression instanceof RexExpression.PatternFieldRef) { + RexExpression.PatternFieldRef ref = (RexExpression.PatternFieldRef) expression; + return ref.withSymbolOrdinal(symbolOrdinals.getOrDefault(ref.getAlpha(), + RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL)); + } + if (!(expression instanceof RexExpression.FunctionCall)) { + return expression; + } + RexExpression.FunctionCall call = (RexExpression.FunctionCall) expression; + List operands = call.getFunctionOperands(); + List boundOperands = new ArrayList<>(operands.size()); + boolean changed = false; + for (RexExpression operand : operands) { + RexExpression bound = bindPatternFieldRefs(operand, symbolOrdinals); + changed |= bound != operand; + boundOperands.add(bound); + } + return changed ? new RexExpression.FunctionCall(call.getDataType(), call.getFunctionName(), boundOperands, + call.isDistinct(), call.isIgnoreNulls()) : call; + } + public static WindowNode.WindowExclusion fromRexWindowExclusion(RexWindowExclusion exclude) { if (exclude == RexWindowExclusion.EXCLUDE_CURRENT_ROW) { return WindowNode.WindowExclusion.CURRENT_ROW; diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpression.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpression.java index 2d4c03a503a2..a6418f4e3ee0 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpression.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpression.java @@ -57,6 +57,87 @@ public int hashCode() { } } + /// Reference to a column of the row bound to a specific MATCH_RECOGNIZE pattern variable, as it appears inside + /// MEASURES and DEFINE expressions. It is the serializable form of Calcite's `RexPatternFieldRef`, and is + /// only meaningful inside a `MatchNode`, whose pattern symbol table the ordinal indexes into. + /// + /// This deliberately does **not** extend [InputRef]. `RexPatternFieldRef` extends + /// `RexInputRef`, so any code that only looks at the index silently turns + /// `DEFINE UP AS UP.price > PREV(UP.price)` into a read of the current row's column: results that are wrong + /// but still type-correct, and therefore invisible. Consumers that do not understand this class must fail loudly. + class PatternFieldRef implements RexExpression { + /// Ordinal used while converting from Calcite, before the pattern symbol table has been built. It must never + /// reach the wire: `RexExpressionToProtoExpression` rejects it. + public static final int UNRESOLVED_SYMBOL_ORDINAL = -1; + /// Ordinal of the SQL:2016 *universal* row pattern variable, i.e. a column reference in MEASURES or DEFINE + /// that is not qualified by a pattern variable, such as `price` in + /// `DEFINE UP AS price > PREV(price)`. It denotes the rows of the match regardless of which variable they + /// are mapped to, so it is **not** an index into the symbol table and must never be confused with one: + /// `LAST(price)` is the last row of the whole match, whereas `LAST(X.price)` is the last row mapped + /// to `X`. + /// + /// Calcite has no dedicated representation for it: it reuses the row source alias (the table alias) as the + /// [alpha][#getAlpha()], which is why the planner maps any alpha that is not a pattern variable onto this + /// ordinal. It is a legal wire value, unlike [#UNRESOLVED_SYMBOL_ORDINAL]. + public static final int UNIVERSAL_SYMBOL_ORDINAL = -2; + + private final int _index; + private final int _symbolOrdinal; + private final String _alpha; + + public PatternFieldRef(int index, int symbolOrdinal, String alpha) { + _index = index; + _symbolOrdinal = symbolOrdinal; + _alpha = alpha; + } + + /// Column index into the input row of the enclosing `MatchNode`. + public int getIndex() { + return _index; + } + + /// Ordinal of the pattern variable, i.e. the index into the enclosing `MatchNode`'s symbol table, or + /// [#UNIVERSAL_SYMBOL_ORDINAL] for an unqualified reference spanning every row of the match. The universal + /// sentinel is the only legal negative wire value; [#UNRESOLVED_SYMBOL_ORDINAL] is planner-local only. + /// This ordinal is the authoritative identification of the variable. + public int getSymbolOrdinal() { + return _symbolOrdinal; + } + + /// Pattern variable name as written in the query, for explain plans and error messages only. Consumers must + /// resolve the variable through [#getSymbolOrdinal()] and never string-match on this value. + public String getAlpha() { + return _alpha; + } + + /// Returns a copy of this reference bound to the given pattern symbol ordinal. + public PatternFieldRef withSymbolOrdinal(int symbolOrdinal) { + return new PatternFieldRef(_index, symbolOrdinal, _alpha); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PatternFieldRef)) { + return false; + } + PatternFieldRef that = (PatternFieldRef) o; + return _index == that._index && _symbolOrdinal == that._symbolOrdinal && Objects.equals(_alpha, that._alpha); + } + + @Override + public int hashCode() { + return Objects.hash(_index, _symbolOrdinal, _alpha); + } + + @Override + public String toString() { + return _alpha + "." + _index; + } + } + class Literal implements RexExpression { public static final Literal TRUE = new Literal(ColumnDataType.BOOLEAN, 1); public static final Literal FALSE = new Literal(ColumnDataType.BOOLEAN, 0); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java index 4458d4ed23aa..1786e3dadb91 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RexExpressionUtils.java @@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexPatternFieldRef; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlIdentifier; @@ -67,7 +68,11 @@ private RexExpressionUtils() { } public static RexNode toRexNode(RelBuilder builder, RexExpression rexExpression) { - if (rexExpression instanceof RexExpression.InputRef) { + // NOTE: PatternFieldRef does not extend InputRef, but it must still be handled before it for symmetry with + // fromRexNode: a pattern field reference that fell through to the InputRef branch would lose its variable. + if (rexExpression instanceof RexExpression.PatternFieldRef) { + return toRexPatternFieldRef(builder, (RexExpression.PatternFieldRef) rexExpression); + } else if (rexExpression instanceof RexExpression.InputRef) { return toRexInputRef(builder, (RexExpression.InputRef) rexExpression); } else if (rexExpression instanceof RexExpression.Literal) { return toRexLiteral(builder, (RexExpression.Literal) rexExpression); @@ -82,6 +87,12 @@ private static RexNode toRexInputRef(RelBuilder builder, RexExpression.InputRef return builder.field(rexExpression.getIndex()); } + private static RexNode toRexPatternFieldRef(RelBuilder builder, RexExpression.PatternFieldRef rexExpression) { + RexNode field = builder.field(rexExpression.getIndex()); + return field instanceof RexInputRef ? RexPatternFieldRef.of(rexExpression.getAlpha(), (RexInputRef) field) + : new RexPatternFieldRef(rexExpression.getAlpha(), rexExpression.getIndex(), field.getType()); + } + private static RexNode toRexCall(RelBuilder builder, RexExpression.FunctionCall rexExpression) { List functionOperands = rexExpression.getFunctionOperands(); List operands = new ArrayList<>(functionOperands.size()); @@ -194,7 +205,12 @@ public static SqlAggFunction getAggFunction(RexExpression.FunctionCall functionC } public static RexExpression fromRexNode(RexNode rexNode) { - if (rexNode instanceof RexInputRef) { + // NOTE: RexPatternFieldRef extends RexInputRef, so it MUST be checked first. Falling through to the RexInputRef + // branch drops the pattern variable and degrades `UP.price` into a read of the current row's column: results that + // are wrong but still type-correct, and therefore invisible. + if (rexNode instanceof RexPatternFieldRef) { + return fromRexPatternFieldRef((RexPatternFieldRef) rexNode); + } else if (rexNode instanceof RexInputRef) { return fromRexInputRef((RexInputRef) rexNode); } else if (rexNode instanceof RexLiteral) { return fromRexLiteral((RexLiteral) rexNode); @@ -217,6 +233,17 @@ public static RexExpression.InputRef fromRexInputRef(RexInputRef rexInputRef) { return new RexExpression.InputRef(rexInputRef.getIndex()); } + /// Converts a MATCH_RECOGNIZE pattern field reference (e.g. `UP.price` inside MEASURES or DEFINE). + /// + /// The pattern symbol table is not known here, so the returned reference carries + /// [RexExpression.PatternFieldRef#UNRESOLVED_SYMBOL_ORDINAL]. The MATCH_RECOGNIZE planning pass must bind it + /// to the symbol ordinal before the plan is serialized; the serializer rejects unresolved ordinals rather than + /// writing an ambiguous reference to the wire. + public static RexExpression.PatternFieldRef fromRexPatternFieldRef(RexPatternFieldRef rexPatternFieldRef) { + return new RexExpression.PatternFieldRef(rexPatternFieldRef.getIndex(), + RexExpression.PatternFieldRef.UNRESOLVED_SYMBOL_ORDINAL, rexPatternFieldRef.getAlpha()); + } + public static RexExpression.Literal fromRexLiteral(RexLiteral rexLiteral) { // TODO: Handle SYMBOL in the planning phase. if (rexLiteral.getTypeName() == SqlTypeName.SYMBOL) { diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/SubPlanFragmenter.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/SubPlanFragmenter.java index d9dcea1f325f..b4da71429787 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/SubPlanFragmenter.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/SubPlanFragmenter.java @@ -33,6 +33,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -120,6 +121,11 @@ public PlanNode visitWindow(WindowNode node, Context context) { return process(node, context); } + @Override + public PlanNode visitMatch(MatchNode node, Context context) { + return process(node, context); + } + @Override public PlanNode visitSetOp(SetOpNode node, Context context) { return process(node, context); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/DispatchablePlanVisitor.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/DispatchablePlanVisitor.java index 5cd307bbebbc..1160508db6b5 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/DispatchablePlanVisitor.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/physical/DispatchablePlanVisitor.java @@ -36,6 +36,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -84,6 +85,17 @@ public Void visitWindow(WindowNode node, DispatchablePlanContext context) { return null; } + @Override + public Void visitMatch(MatchNode node, DispatchablePlanContext context) { + node.getInputs().get(0).visit(this, context); + DispatchablePlanMetadata dispatchablePlanMetadata = getOrCreateDispatchablePlanMetadata(node, context); + // Same reasoning as WindowNode: MATCH_RECOGNIZE without PARTITION BY sees the whole input as a single ordered + // partition, so it has to run on a singleton node. With PARTITION BY the work can be distributed, since matches + // never span partitions. + dispatchablePlanMetadata.setRequireSingleton(node.getPartitionKeys().isEmpty()); + return null; + } + @Override public Void visitSetOp(SetOpNode setOpNode, DispatchablePlanContext context) { setOpNode.getInputs().forEach(input -> input.visit(this, context)); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/DefaultPostOrderTraversalVisitor.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/DefaultPostOrderTraversalVisitor.java index 26abf9ed2e18..67ba7adc12df 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/DefaultPostOrderTraversalVisitor.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/DefaultPostOrderTraversalVisitor.java @@ -89,6 +89,12 @@ public T visitWindow(WindowNode node, C context) { return process(node, context); } + @Override + public T visitMatch(MatchNode node, C context) { + node.getInputs().get(0).visit(this, context); + return process(node, context); + } + @Override public T visitSetOp(SetOpNode node, C context) { node.getInputs().forEach(input -> input.visit(this, context)); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/MatchNode.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/MatchNode.java new file mode 100644 index 000000000000..9fc9a1701560 --- /dev/null +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/MatchNode.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.plannode; + +import java.util.List; +import java.util.Objects; +import javax.annotation.Nullable; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.query.planner.logical.RexExpression; + + +/// MatchNode models SQL:2016 MATCH_RECOGNIZE (row pattern recognition): it partitions its input, orders each +/// partition, finds the runs of rows that match a row pattern, and emits the MEASURES of every match. +/// +/// The pattern is carried as a self-contained [RowPattern] tree over a symbol table +/// ([#getPatternSymbols()]), rather than as Calcite's `RexCall`-over-string-literals encoding. Every +/// reference to a pattern variable - [RowPattern.Symbol], [RexExpression.PatternFieldRef] and +/// [#getAfterMatchSkipToSymbolOrdinal()] - is an ordinal into that table, so the operator never string-matches +/// variable names. +/// +/// DataSchema on this node reflects the output schema: the PARTITION BY columns followed by the MEASURES columns +/// for [RowsPerMatchMode#ONE_ROW_PER_MATCH]. +/// +/// Not supported in this node and rejected at planning time: `SUBSET`, `PERMUTE`, `{- -}` +/// exclusions, `WITHIN`, `OMIT EMPTY MATCHES`, `WITH UNMATCHED ROWS`, aggregates inside +/// `DEFINE`, and [RowsPerMatchMode#ALL_ROWS_PER_MATCH]. +public class MatchNode extends BasePlanNode { + /// [#getAfterMatchSkipToSymbolOrdinal()] value for the skip modes that have no target variable. + public static final int NO_SKIP_TO_SYMBOL = -1; + + private final List _patternSymbols; + private final RowPattern _pattern; + private final List _measures; + private final List _partitionKeys; + private final List _collations; + private final AfterMatchSkipMode _afterMatchSkipMode; + private final int _afterMatchSkipToSymbolOrdinal; + private final RowsPerMatchMode _rowsPerMatchMode; + + public MatchNode(int stageId, DataSchema dataSchema, NodeHint nodeHint, List inputs, + List patternSymbols, RowPattern pattern, List measures, List partitionKeys, + List collations, AfterMatchSkipMode afterMatchSkipMode, int afterMatchSkipToSymbolOrdinal, + RowsPerMatchMode rowsPerMatchMode) { + super(stageId, dataSchema, nodeHint, inputs); + _patternSymbols = List.copyOf(patternSymbols); + _pattern = pattern; + _measures = List.copyOf(measures); + _partitionKeys = List.copyOf(partitionKeys); + _collations = List.copyOf(collations); + _afterMatchSkipMode = afterMatchSkipMode; + _afterMatchSkipToSymbolOrdinal = afterMatchSkipToSymbolOrdinal; + _rowsPerMatchMode = rowsPerMatchMode; + validatePatternFieldRefs(_patternSymbols, _measures); + } + + private static void validatePatternFieldRefs(List patternSymbols, List measures) { + int numSymbols = patternSymbols.size(); + for (PatternSymbol symbol : patternSymbols) { + validatePatternFieldRefs(symbol.getDefinition(), numSymbols); + } + for (Measure measure : measures) { + validatePatternFieldRefs(measure.getExpression(), numSymbols); + } + } + + private static void validatePatternFieldRefs(@Nullable RexExpression expression, int numSymbols) { + if (expression == null) { + return; + } + if (expression instanceof RexExpression.PatternFieldRef) { + RexExpression.PatternFieldRef ref = (RexExpression.PatternFieldRef) expression; + int ordinal = ref.getSymbolOrdinal(); + if (ordinal != RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL + && (ordinal < 0 || ordinal >= numSymbols)) { + throw new IllegalArgumentException( + "MATCH_RECOGNIZE pattern field reference '" + ref + "' has invalid symbol ordinal " + ordinal + + "; expected the universal ordinal " + RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL + + " or an index in [0, " + numSymbols + ")"); + } + return; + } + if (expression instanceof RexExpression.FunctionCall) { + for (RexExpression operand : ((RexExpression.FunctionCall) expression).getFunctionOperands()) { + validatePatternFieldRefs(operand, numSymbols); + } + } + } + + /// The pattern variable symbol table. A symbol's ordinal is its index in this list. + public List getPatternSymbols() { + return _patternSymbols; + } + + /// Root of the row pattern tree, i.e. the `PATTERN` clause. + public RowPattern getPattern() { + return _pattern; + } + + /// The `MEASURES` items, in output order. + public List getMeasures() { + return _measures; + } + + /// `PARTITION BY`: column indexes into the input row. Empty means a single partition over the whole input. + public List getPartitionKeys() { + return _partitionKeys; + } + + /// `ORDER BY`. Mandatory for MATCH_RECOGNIZE, so this is never empty. + public List getCollations() { + return _collations; + } + + public AfterMatchSkipMode getAfterMatchSkipMode() { + return _afterMatchSkipMode; + } + + /// For [AfterMatchSkipMode#TO_FIRST] and [AfterMatchSkipMode#TO_LAST], the ordinal of the target + /// pattern variable; [#NO_SKIP_TO_SYMBOL] for the other skip modes. + public int getAfterMatchSkipToSymbolOrdinal() { + return _afterMatchSkipToSymbolOrdinal; + } + + public RowsPerMatchMode getRowsPerMatchMode() { + return _rowsPerMatchMode; + } + + /// Renders the pattern using the symbol table, e.g. `(A B* | C){2,3}`, for explain plans and error messages. + public String getPatternString() { + StringBuilder builder = new StringBuilder(); + _pattern.appendTo(builder, _patternSymbols); + return builder.toString(); + } + + @Override + public String explain() { + return "MATCH_RECOGNIZE"; + } + + @Override + public T visit(PlanNodeVisitor visitor, C context) { + return visitor.visitMatch(this, context); + } + + @Override + public PlanNode withInputs(List inputs) { + return new MatchNode(_stageId, _dataSchema, _nodeHint, inputs, _patternSymbols, _pattern, _measures, + _partitionKeys, _collations, _afterMatchSkipMode, _afterMatchSkipToSymbolOrdinal, _rowsPerMatchMode); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MatchNode)) { + return false; + } + if (!super.equals(o)) { + return false; + } + MatchNode that = (MatchNode) o; + return _afterMatchSkipToSymbolOrdinal == that._afterMatchSkipToSymbolOrdinal + && _afterMatchSkipMode == that._afterMatchSkipMode && _rowsPerMatchMode == that._rowsPerMatchMode + && Objects.equals(_patternSymbols, that._patternSymbols) && Objects.equals(_pattern, that._pattern) + && Objects.equals(_measures, that._measures) && Objects.equals(_partitionKeys, that._partitionKeys) + && Objects.equals(_collations, that._collations); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), _patternSymbols, _pattern, _measures, _partitionKeys, _collations, + _afterMatchSkipMode, _afterMatchSkipToSymbolOrdinal, _rowsPerMatchMode); + } + + /// One `MEASURES AS ` item. + public static final class Measure { + private final String _name; + private final RexExpression _expression; + + public Measure(String name, RexExpression expression) { + _name = name; + _expression = expression; + } + + /// Output column name of this measure. + public String getName() { + return _name; + } + + public RexExpression getExpression() { + return _expression; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Measure)) { + return false; + } + Measure that = (Measure) o; + return Objects.equals(_name, that._name) && Objects.equals(_expression, that._expression); + } + + @Override + public int hashCode() { + return Objects.hash(_name, _expression); + } + + @Override + public String toString() { + return _expression + " AS " + _name; + } + } + + /// SQL:2016 `AFTER MATCH SKIP` clause: where pattern matching resumes after a successful match. + /// + /// [#PAST_LAST_ROW] is the SQL:2016 default (and the default in Trino, Snowflake and Oracle), which + /// produces non-overlapping matches. Calcite defaults an omitted clause to [#TO_NEXT_ROW] instead, which + /// produces overlapping matches, so the default must be fixed at the `SqlNode` level before conversion. + /// + /// The constant names are part of the wire protocol via `Plan.AfterMatchSkipMode` and must remain stable + /// across mixed-version brokers and servers. + public enum AfterMatchSkipMode { + /// Resume at the row after the last row of the match. + PAST_LAST_ROW, + /// Resume at the row after the first row of the match, so matches may overlap. + TO_NEXT_ROW, + /// Resume at the first row mapped to the target pattern variable. + TO_FIRST, + /// Resume at the last row mapped to the target pattern variable. + TO_LAST + } + + /// SQL:2016 `ONE ROW PER MATCH` / `ALL ROWS PER MATCH`. + /// + /// [#ALL_ROWS_PER_MATCH] is declared so its wire value is pinned and so the planner can reject it by + /// name, but it is not supported yet. + /// + /// The constant names are part of the wire protocol via `Plan.RowsPerMatchMode` and must remain stable + /// across mixed-version brokers and servers. + public enum RowsPerMatchMode { + ONE_ROW_PER_MATCH, ALL_ROWS_PER_MATCH + } +} diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PatternSymbol.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PatternSymbol.java new file mode 100644 index 000000000000..fb2bf6c03925 --- /dev/null +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PatternSymbol.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.plannode; + +import java.util.Objects; +import javax.annotation.Nullable; +import org.apache.pinot.query.planner.logical.RexExpression; + + +/// One entry of a [MatchNode]'s MATCH_RECOGNIZE pattern variable symbol table: a variable name and its +/// `DEFINE` predicate. +/// +/// The variable's *ordinal* is its index in [MatchNode#getPatternSymbols()]. Everything that refers +/// to a pattern variable - [RowPattern.Symbol], [RexExpression.PatternFieldRef] and +/// [MatchNode#getAfterMatchSkipToSymbolOrdinal()] - does so by that ordinal, so the operator never has to +/// string-match on [#getName()]. +/// +/// Instances are immutable. +public class PatternSymbol { + private final String _name; + private final RexExpression _definition; + + public PatternSymbol(String name, @Nullable RexExpression definition) { + _name = name; + _definition = definition; + } + + /// Pattern variable name as written in the `PATTERN` / `DEFINE` clauses. Informational only. + public String getName() { + return _name; + } + + /// The `DEFINE` predicate for this variable, or `null` when the variable has no `DEFINE` entry. + /// Per SQL:2016 an undefined pattern variable matches every row. + @Nullable + public RexExpression getDefinition() { + return _definition; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PatternSymbol)) { + return false; + } + PatternSymbol that = (PatternSymbol) o; + return Objects.equals(_name, that._name) && Objects.equals(_definition, that._definition); + } + + @Override + public int hashCode() { + return Objects.hash(_name, _definition); + } + + @Override + public String toString() { + return _definition != null ? _name + " AS " + _definition : _name; + } +} diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNodeVisitor.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNodeVisitor.java index 47aa7aa56d4a..ab25669ad771 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNodeVisitor.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNodeVisitor.java @@ -59,6 +59,8 @@ public interface PlanNodeVisitor { T visitWindow(WindowNode node, C context); + T visitMatch(MatchNode node, C context); + T visitSetOp(SetOpNode node, C context); T visitExchange(ExchangeNode node, C context); @@ -213,6 +215,13 @@ public T visitWindow(WindowNode node, C context) { return postChildren(node, context); } + @Override + public T visitMatch(MatchNode node, C context) { + preChildren(node, context); + visitChildren(node, context); + return postChildren(node, context); + } + @Override public T visitSetOp(SetOpNode node, C context) { preChildren(node, context); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/RowPattern.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/RowPattern.java new file mode 100644 index 000000000000..2158a3166d35 --- /dev/null +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/RowPattern.java @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.plannode; + +import java.util.List; +import java.util.Objects; + + +/// A node of the MATCH_RECOGNIZE row pattern tree, i.e. the parsed form of the `PATTERN` clause. +/// +/// This is a self-contained representation: pattern variables are referenced by their ordinal in the enclosing +/// [MatchNode]'s symbol table, never by name, so an operator never has to string-match variable names. It is +/// also independent of Calcite's encoding of the pattern as a `RexCall` tree over string literals. +/// +/// Instances are immutable. +public interface RowPattern { + + Kind getKind(); + + /// Renders this pattern into `builder` using `symbols` to resolve variable ordinals to names, e.g. + /// `(A B* | C){2,3}`. Used for explain plans and error messages. + void appendTo(StringBuilder builder, List symbols); + + /// The kind of pattern node. + /// + /// The SQL:2016 `{- -}` exclusion and `PERMUTE(...)` constructs are deliberately absent: they are + /// rejected at planning time. Their wire values are already pinned in `Plan.RowPatternKind` so that adding + /// them later is a purely additive change. + enum Kind { + SYMBOL, CONCAT, ALTERNATE, QUANTIFIER, ANCHOR_START, ANCHOR_END + } + + /// A single pattern variable, identified by its ordinal in the enclosing [MatchNode]'s symbol table. + final class Symbol implements RowPattern { + private final int _symbolOrdinal; + + public Symbol(int symbolOrdinal) { + _symbolOrdinal = symbolOrdinal; + } + + public int getSymbolOrdinal() { + return _symbolOrdinal; + } + + @Override + public Kind getKind() { + return Kind.SYMBOL; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + builder.append( + _symbolOrdinal >= 0 && _symbolOrdinal < symbols.size() ? symbols.get(_symbolOrdinal).getName() + : "?" + _symbolOrdinal); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Symbol)) { + return false; + } + return _symbolOrdinal == ((Symbol) o)._symbolOrdinal; + } + + @Override + public int hashCode() { + return Objects.hash(Kind.SYMBOL, _symbolOrdinal); + } + } + + /// Juxtaposition of two or more sub-patterns that must match consecutively, e.g. `A B C`. + final class Concat implements RowPattern { + private final List _children; + + public Concat(List children) { + _children = List.copyOf(children); + } + + public List getChildren() { + return _children; + } + + @Override + public Kind getKind() { + return Kind.CONCAT; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + for (int i = 0; i < _children.size(); i++) { + if (i > 0) { + builder.append(' '); + } + _children.get(i).appendTo(builder, symbols); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Concat)) { + return false; + } + return _children.equals(((Concat) o)._children); + } + + @Override + public int hashCode() { + return Objects.hash(Kind.CONCAT, _children); + } + } + + /// Alternation of two or more sub-patterns, e.g. `A | B`. Alternatives are ordered: per SQL:2016 the + /// leftmost alternative that yields a match wins. + final class Alternate implements RowPattern { + private final List _children; + + public Alternate(List children) { + _children = List.copyOf(children); + } + + public List getChildren() { + return _children; + } + + @Override + public Kind getKind() { + return Kind.ALTERNATE; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + builder.append('('); + for (int i = 0; i < _children.size(); i++) { + if (i > 0) { + builder.append(" | "); + } + _children.get(i).appendTo(builder, symbols); + } + builder.append(')'); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Alternate)) { + return false; + } + return _children.equals(((Alternate) o)._children); + } + + @Override + public int hashCode() { + return Objects.hash(Kind.ALTERNATE, _children); + } + } + + /// A repetition applied to a single sub-pattern. `*`, `+` and `?` are normalized into this form: + /// `*` is `{0,UNBOUNDED}`, `+` is `{1,UNBOUNDED}` and `?` is `{0,1}`. + final class Quantifier implements RowPattern { + /// Sentinel for `maxRepeat` meaning "no upper bound", i.e. `*`, `+` and `{n,}`. + public static final int UNBOUNDED = -1; + + private final RowPattern _child; + private final int _minRepeat; + private final int _maxRepeat; + private final boolean _greedy; + + public Quantifier(RowPattern child, int minRepeat, int maxRepeat, boolean greedy) { + _child = child; + _minRepeat = minRepeat; + _maxRepeat = maxRepeat; + _greedy = greedy; + } + + public RowPattern getChild() { + return _child; + } + + public int getMinRepeat() { + return _minRepeat; + } + + /// Maximum number of repetitions, or [#UNBOUNDED]. + public int getMaxRepeat() { + return _maxRepeat; + } + + /// Whether the quantifier is greedy. The reluctant forms (`*?`, `+?`, `??`, `{n,m}?`) + /// prefer the shortest match. + public boolean isGreedy() { + return _greedy; + } + + @Override + public Kind getKind() { + return Kind.QUANTIFIER; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + // Concat binds looser than a quantifier, so it has to be parenthesized. Alternate parenthesizes itself. + boolean parenthesize = _child.getKind() == Kind.CONCAT; + if (parenthesize) { + builder.append('('); + } + _child.appendTo(builder, symbols); + if (parenthesize) { + builder.append(')'); + } + appendQuantifier(builder); + if (!_greedy) { + builder.append('?'); + } + } + + private void appendQuantifier(StringBuilder builder) { + if (_minRepeat == 0 && _maxRepeat == UNBOUNDED) { + builder.append('*'); + } else if (_minRepeat == 1 && _maxRepeat == UNBOUNDED) { + builder.append('+'); + } else if (_minRepeat == 0 && _maxRepeat == 1) { + builder.append('?'); + } else if (_maxRepeat == UNBOUNDED) { + builder.append('{').append(_minRepeat).append(",}"); + } else if (_minRepeat == _maxRepeat) { + builder.append('{').append(_minRepeat).append('}'); + } else { + builder.append('{').append(_minRepeat).append(',').append(_maxRepeat).append('}'); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Quantifier)) { + return false; + } + Quantifier that = (Quantifier) o; + return _minRepeat == that._minRepeat && _maxRepeat == that._maxRepeat && _greedy == that._greedy + && _child.equals(that._child); + } + + @Override + public int hashCode() { + return Objects.hash(Kind.QUANTIFIER, _child, _minRepeat, _maxRepeat, _greedy); + } + } + + /// The `^` anchor: the match must start at the first row of the partition. + final class AnchorStart implements RowPattern { + public static final AnchorStart INSTANCE = new AnchorStart(); + + private AnchorStart() { + } + + @Override + public Kind getKind() { + return Kind.ANCHOR_START; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + builder.append('^'); + } + + @Override + public boolean equals(Object o) { + return o instanceof AnchorStart; + } + + @Override + public int hashCode() { + return Kind.ANCHOR_START.hashCode(); + } + } + + /// The `$` anchor: the match must end at the last row of the partition. + final class AnchorEnd implements RowPattern { + public static final AnchorEnd INSTANCE = new AnchorEnd(); + + private AnchorEnd() { + } + + @Override + public Kind getKind() { + return Kind.ANCHOR_END; + } + + @Override + public void appendTo(StringBuilder builder, List symbols) { + builder.append('$'); + } + + @Override + public boolean equals(Object o) { + return o instanceof AnchorEnd; + } + + @Override + public int hashCode() { + return Kind.ANCHOR_END.hashCode(); + } + } +} diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java index 9ad7f841de0d..1a5238965211 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java @@ -40,8 +40,11 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.ProjectNode; +import org.apache.pinot.query.planner.plannode.RowPattern; import org.apache.pinot.query.planner.plannode.SetOpNode; import org.apache.pinot.query.planner.plannode.SortNode; import org.apache.pinot.query.planner.plannode.TableScanNode; @@ -87,6 +90,8 @@ public static PlanNode process(Plan.PlanNode protoNode) { return deserializeEnrichedJoinNode(protoNode); case UNNESTNODE: return deserializeUnnestNode(protoNode); + case MATCHNODE: + return deserializeMatchNode(protoNode); default: throw new IllegalStateException("Unsupported PlanNode type: " + protoNode.getNodeCase()); } @@ -264,6 +269,96 @@ private static UnnestNode deserializeUnnestNode(Plan.PlanNode protoNode) { extractInputs(protoNode), arrayExprs, context); } + private static MatchNode deserializeMatchNode(Plan.PlanNode protoNode) { + Plan.MatchNode protoMatchNode = protoNode.getMatchNode(); + + List protoSymbols = protoMatchNode.getPatternSymbolsList(); + List patternSymbols = new ArrayList<>(protoSymbols.size()); + for (Plan.PatternSymbol protoSymbol : protoSymbols) { + // An absent definition means the pattern variable has no DEFINE entry, i.e. it matches every row. + patternSymbols.add(new PatternSymbol(protoSymbol.getName(), + protoSymbol.hasDefinition() ? ProtoExpressionToRexExpression.convertExpression(protoSymbol.getDefinition()) + : null)); + } + + List protoMeasures = protoMatchNode.getMeasuresList(); + List measures = new ArrayList<>(protoMeasures.size()); + for (Plan.MatchMeasure protoMeasure : protoMeasures) { + measures.add(new MatchNode.Measure(protoMeasure.getName(), + ProtoExpressionToRexExpression.convertExpression(protoMeasure.getExpression()))); + } + + int skipToSymbolOrdinal = protoMatchNode.hasAfterMatchSkipToSymbolOrdinal() + ? protoMatchNode.getAfterMatchSkipToSymbolOrdinal() : MatchNode.NO_SKIP_TO_SYMBOL; + + return new MatchNode(protoNode.getStageId(), extractDataSchema(protoNode), extractNodeHint(protoNode), + extractInputs(protoNode), patternSymbols, convertRowPattern(protoMatchNode.getPattern()), measures, + protoMatchNode.getPartitionKeysList(), convertCollations(protoMatchNode.getCollationsList()), + convertAfterMatchSkipMode(protoMatchNode.getAfterMatchSkipMode()), skipToSymbolOrdinal, + convertRowsPerMatchMode(protoMatchNode.getRowsPerMatchMode())); + } + + private static RowPattern convertRowPattern(Plan.RowPattern protoPattern) { + List protoChildren = protoPattern.getChildrenList(); + switch (protoPattern.getKind()) { + case PATTERN_SYMBOL: + return new RowPattern.Symbol(protoPattern.getSymbolOrdinal()); + case PATTERN_CONCAT: + return new RowPattern.Concat(convertRowPatterns(protoChildren)); + case PATTERN_ALTERNATE: + return new RowPattern.Alternate(convertRowPatterns(protoChildren)); + case PATTERN_QUANTIFIER: { + Preconditions.checkState(protoChildren.size() == 1, + "PATTERN_QUANTIFIER must have exactly 1 child, got: %s", protoChildren.size()); + Plan.RowPatternQuantifier quantifier = protoPattern.getQuantifier(); + return new RowPattern.Quantifier(convertRowPattern(protoChildren.get(0)), quantifier.getMinRepeat(), + quantifier.getMaxRepeat(), quantifier.getGreedy()); + } + case PATTERN_ANCHOR_START: + return RowPattern.AnchorStart.INSTANCE; + case PATTERN_ANCHOR_END: + return RowPattern.AnchorEnd.INSTANCE; + default: + // PATTERN_EXCLUDE and PATTERN_PERMUTE are pinned in the wire format but not implemented. A newer broker + // must not be allowed to degrade into a different pattern on an older server. + throw new IllegalStateException("Unsupported MATCH_RECOGNIZE pattern kind: " + protoPattern.getKind()); + } + } + + private static List convertRowPatterns(List protoPatterns) { + List patterns = new ArrayList<>(protoPatterns.size()); + for (Plan.RowPattern protoPattern : protoPatterns) { + patterns.add(convertRowPattern(protoPattern)); + } + return patterns; + } + + private static MatchNode.AfterMatchSkipMode convertAfterMatchSkipMode(Plan.AfterMatchSkipMode skipMode) { + switch (skipMode) { + case SKIP_PAST_LAST_ROW: + return MatchNode.AfterMatchSkipMode.PAST_LAST_ROW; + case SKIP_TO_NEXT_ROW: + return MatchNode.AfterMatchSkipMode.TO_NEXT_ROW; + case SKIP_TO_FIRST: + return MatchNode.AfterMatchSkipMode.TO_FIRST; + case SKIP_TO_LAST: + return MatchNode.AfterMatchSkipMode.TO_LAST; + default: + throw new IllegalStateException("Unsupported AfterMatchSkipMode: " + skipMode); + } + } + + private static MatchNode.RowsPerMatchMode convertRowsPerMatchMode(Plan.RowsPerMatchMode rowsPerMatchMode) { + switch (rowsPerMatchMode) { + case ONE_ROW_PER_MATCH: + return MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH; + case ALL_ROWS_PER_MATCH: + return MatchNode.RowsPerMatchMode.ALL_ROWS_PER_MATCH; + default: + throw new IllegalStateException("Unsupported RowsPerMatchMode: " + rowsPerMatchMode); + } + } + private static DataSchema extractDataSchema(Plan.DataSchema protoDataSchema) { String[] columnNames = protoDataSchema.getColumnNamesList().toArray(new String[0]); int numColumns = columnNames.length; diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java index 81dd73cf41fc..95671c69a9aa 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java @@ -38,10 +38,13 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNode.NodeHint; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; +import org.apache.pinot.query.planner.plannode.RowPattern; import org.apache.pinot.query.planner.plannode.SetOpNode; import org.apache.pinot.query.planner.plannode.SortNode; import org.apache.pinot.query.planner.plannode.TableScanNode; @@ -298,6 +301,106 @@ public Void visitUnnest(UnnestNode node, Plan.PlanNode.Builder builder) { return null; } + @Override + public Void visitMatch(MatchNode node, Plan.PlanNode.Builder builder) { + Plan.MatchNode.Builder matchNodeBuilder = Plan.MatchNode.newBuilder() + .setPattern(convertRowPattern(node.getPattern())) + .addAllPartitionKeys(node.getPartitionKeys()) + .addAllCollations(convertCollations(node.getCollations())) + .setAfterMatchSkipMode(convertAfterMatchSkipMode(node.getAfterMatchSkipMode())) + .setRowsPerMatchMode(convertRowsPerMatchMode(node.getRowsPerMatchMode())); + for (PatternSymbol symbol : node.getPatternSymbols()) { + Plan.PatternSymbol.Builder symbolBuilder = Plan.PatternSymbol.newBuilder().setName(symbol.getName()); + RexExpression definition = symbol.getDefinition(); + if (definition != null) { + symbolBuilder.setDefinition(RexExpressionToProtoExpression.convertExpression(definition)); + } + matchNodeBuilder.addPatternSymbols(symbolBuilder.build()); + } + for (MatchNode.Measure measure : node.getMeasures()) { + matchNodeBuilder.addMeasures(Plan.MatchMeasure.newBuilder() + .setName(measure.getName()) + .setExpression(RexExpressionToProtoExpression.convertExpression(measure.getExpression())) + .build()); + } + // Left unset for the skip modes without a target variable, because ordinal 0 is itself a valid variable. + if (node.getAfterMatchSkipToSymbolOrdinal() != MatchNode.NO_SKIP_TO_SYMBOL) { + matchNodeBuilder.setAfterMatchSkipToSymbolOrdinal(node.getAfterMatchSkipToSymbolOrdinal()); + } + builder.setMatchNode(matchNodeBuilder.build()); + return null; + } + + private static Plan.RowPattern convertRowPattern(RowPattern pattern) { + Plan.RowPattern.Builder builder = Plan.RowPattern.newBuilder(); + switch (pattern.getKind()) { + case SYMBOL: + builder.setKind(Plan.RowPatternKind.PATTERN_SYMBOL) + .setSymbolOrdinal(((RowPattern.Symbol) pattern).getSymbolOrdinal()); + break; + case CONCAT: + builder.setKind(Plan.RowPatternKind.PATTERN_CONCAT); + addRowPatternChildren(builder, ((RowPattern.Concat) pattern).getChildren()); + break; + case ALTERNATE: + builder.setKind(Plan.RowPatternKind.PATTERN_ALTERNATE); + addRowPatternChildren(builder, ((RowPattern.Alternate) pattern).getChildren()); + break; + case QUANTIFIER: { + RowPattern.Quantifier quantifier = (RowPattern.Quantifier) pattern; + builder.setKind(Plan.RowPatternKind.PATTERN_QUANTIFIER) + .addChildren(convertRowPattern(quantifier.getChild())) + .setQuantifier(Plan.RowPatternQuantifier.newBuilder() + .setMinRepeat(quantifier.getMinRepeat()) + .setMaxRepeat(quantifier.getMaxRepeat()) + .setGreedy(quantifier.isGreedy()) + .build()); + break; + } + case ANCHOR_START: + builder.setKind(Plan.RowPatternKind.PATTERN_ANCHOR_START); + break; + case ANCHOR_END: + builder.setKind(Plan.RowPatternKind.PATTERN_ANCHOR_END); + break; + default: + throw new IllegalStateException("Unsupported RowPattern kind: " + pattern.getKind()); + } + return builder.build(); + } + + private static void addRowPatternChildren(Plan.RowPattern.Builder builder, List children) { + for (RowPattern child : children) { + builder.addChildren(convertRowPattern(child)); + } + } + + private static Plan.AfterMatchSkipMode convertAfterMatchSkipMode(MatchNode.AfterMatchSkipMode skipMode) { + switch (skipMode) { + case PAST_LAST_ROW: + return Plan.AfterMatchSkipMode.SKIP_PAST_LAST_ROW; + case TO_NEXT_ROW: + return Plan.AfterMatchSkipMode.SKIP_TO_NEXT_ROW; + case TO_FIRST: + return Plan.AfterMatchSkipMode.SKIP_TO_FIRST; + case TO_LAST: + return Plan.AfterMatchSkipMode.SKIP_TO_LAST; + default: + throw new IllegalStateException("Unsupported AfterMatchSkipMode: " + skipMode); + } + } + + private static Plan.RowsPerMatchMode convertRowsPerMatchMode(MatchNode.RowsPerMatchMode rowsPerMatchMode) { + switch (rowsPerMatchMode) { + case ONE_ROW_PER_MATCH: + return Plan.RowsPerMatchMode.ONE_ROW_PER_MATCH; + case ALL_ROWS_PER_MATCH: + return Plan.RowsPerMatchMode.ALL_ROWS_PER_MATCH; + default: + throw new IllegalStateException("Unsupported RowsPerMatchMode: " + rowsPerMatchMode); + } + } + private static List convertExpressions(List expressions) { List protoExpressions = new ArrayList<>(expressions.size()); for (RexExpression expression : expressions) { diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java index 060ca8447214..d39cc61a55ef 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/ProtoExpressionToRexExpression.java @@ -41,6 +41,8 @@ public static RexExpression convertExpression(Expressions.Expression expression) return convertLiteral(expression.getLiteral()); case FUNCTIONCALL: return convertFunctionCall(expression.getFunctionCall()); + case PATTERNFIELDREF: + return convertPatternFieldRef(expression.getPatternFieldRef()); default: throw new IllegalStateException("Unsupported proto Expression type: " + expression.getExpressionCase()); } @@ -50,6 +52,18 @@ public static RexExpression.InputRef convertInputRef(Expressions.InputRef inputR return new RexExpression.InputRef(inputRef.getIndex()); } + public static RexExpression.PatternFieldRef convertPatternFieldRef(Expressions.PatternFieldRef patternFieldRef) { + int symbolOrdinal = patternFieldRef.getSymbolOrdinal(); + if (symbolOrdinal == RexExpression.PatternFieldRef.UNRESOLVED_SYMBOL_ORDINAL + || symbolOrdinal < RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + throw new IllegalStateException( + "Invalid MATCH_RECOGNIZE pattern variable ordinal on the wire: " + symbolOrdinal + + ". Expected a non-negative symbol-table index or the universal ordinal " + + RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL + "."); + } + return new RexExpression.PatternFieldRef(patternFieldRef.getIndex(), symbolOrdinal, patternFieldRef.getAlpha()); + } + public static RexExpression.FunctionCall convertFunctionCall(Expressions.FunctionCall functionCall) { List protoOperands = functionCall.getFunctionOperandsList(); List operands = new ArrayList<>(protoOperands.size()); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java index 7b0c79ec5d81..5f41126bcec6 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/RexExpressionToProtoExpression.java @@ -45,9 +45,14 @@ public static Expressions.Expression convertExpression(RexExpression expression) expressionBuilder.setInputRef(convertInputRef((RexExpression.InputRef) expression)); } else if (expression instanceof RexExpression.Literal) { expressionBuilder.setLiteral(convertLiteral((RexExpression.Literal) expression)); - } else { - assert expression instanceof RexExpression.FunctionCall; + } else if (expression instanceof RexExpression.PatternFieldRef) { + expressionBuilder.setPatternFieldRef(convertPatternFieldRef((RexExpression.PatternFieldRef) expression)); + } else if (expression instanceof RexExpression.FunctionCall) { expressionBuilder.setFunctionCall(convertFunctionCall((RexExpression.FunctionCall) expression)); + } else { + // Never silently drop an expression kind: an unhandled kind that fell through to InputRef here would produce + // results that are wrong but type-correct. + throw new IllegalStateException("Unsupported RexExpression type: " + expression.getClass().getName()); } return expressionBuilder.build(); } @@ -56,6 +61,29 @@ public static Expressions.InputRef convertInputRef(RexExpression.InputRef inputR return Expressions.InputRef.newBuilder().setIndex(inputRef.getIndex()).build(); } + /// Converts a MATCH_RECOGNIZE pattern field reference. Only a non-negative symbol-table index or the universal + /// ordinal is legal on the wire; the MATCH_RECOGNIZE planning pass must bind every unresolved reference before the + /// plan is serialized. + public static Expressions.PatternFieldRef convertPatternFieldRef(RexExpression.PatternFieldRef patternFieldRef) { + int symbolOrdinal = patternFieldRef.getSymbolOrdinal(); + if (symbolOrdinal == RexExpression.PatternFieldRef.UNRESOLVED_SYMBOL_ORDINAL) { + throw new IllegalStateException( + "Unresolved MATCH_RECOGNIZE pattern variable in reference to: " + patternFieldRef.getAlpha() + "." + + patternFieldRef.getIndex() + ". The pattern symbol table must be bound before serialization."); + } + if (symbolOrdinal < RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + throw new IllegalStateException( + "Invalid MATCH_RECOGNIZE pattern variable ordinal " + symbolOrdinal + " in reference to: " + + patternFieldRef + ". Expected a non-negative symbol-table index or the universal ordinal " + + RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL + "."); + } + return Expressions.PatternFieldRef.newBuilder() + .setIndex(patternFieldRef.getIndex()) + .setSymbolOrdinal(symbolOrdinal) + .setAlpha(patternFieldRef.getAlpha()) + .build(); + } + public static Expressions.FunctionCall convertFunctionCall(RexExpression.FunctionCall functionCall) { List operands = functionCall.getFunctionOperands(); List protoOperands = new ArrayList<>(operands.size()); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/validation/ArrayToMvValidationVisitor.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/validation/ArrayToMvValidationVisitor.java index 6653f1f22fcf..1748ab42d132 100644 --- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/validation/ArrayToMvValidationVisitor.java +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/validation/ArrayToMvValidationVisitor.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.query.planner.validation; +import java.util.ArrayList; import java.util.List; import org.apache.pinot.common.utils.DataSchema; import org.apache.pinot.query.planner.logical.RexExpression; @@ -29,6 +30,8 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; import org.apache.pinot.query.planner.plannode.SetOpNode; @@ -147,6 +150,28 @@ public Void visitWindow(WindowNode node, Boolean isIntermediateStage) { return null; } + @Override + public Void visitMatch(MatchNode node, Boolean isIntermediateStage) { + if (isIntermediateStage) { + // MEASURES expressions and DEFINE predicates are both evaluated by the MATCH_RECOGNIZE operator itself. + List expressions = new ArrayList<>(node.getMeasures().size() + node.getPatternSymbols().size()); + for (MatchNode.Measure measure : node.getMeasures()) { + expressions.add(measure.getExpression()); + } + for (PatternSymbol symbol : node.getPatternSymbols()) { + if (symbol.getDefinition() != null) { + expressions.add(symbol.getDefinition()); + } + } + if (containsArrayToMv(expressions)) { + throw new QueryException(QueryErrorCode.QUERY_PLANNING, + "Function 'ArrayToMv' is not supported in MATCH_RECOGNIZE Intermediate Stage"); + } + } + node.getInputs().forEach(e -> e.visit(this, isIntermediateStage)); + return null; + } + @Override public Void visitSetOp(SetOpNode setOpNode, Boolean isIntermediateStage) { setOpNode.getInputs().forEach(e -> e.visit(this, isIntermediateStage)); diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/validate/MatchRecognizeValidator.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/validate/MatchRecognizeValidator.java new file mode 100644 index 000000000000..a3dd95d47c7d --- /dev/null +++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/validate/MatchRecognizeValidator.java @@ -0,0 +1,419 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.validate; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlUnresolvedFunction; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/// Pinot specific validation and rewriting of SQL:2016 `MATCH_RECOGNIZE` (row pattern recognition) clauses. +/// +/// This visitor runs on the **raw parsed** [SqlNode] tree, before Calcite's own validation and before +/// [org.apache.calcite.sql2rel.SqlToRelConverter] runs. Running this early is deliberate and load bearing: +/// +/// 1. **The `AFTER MATCH` default must be corrected at the `SqlNode` level.** When the `AFTER MATCH` clause is +/// omitted, the parser leaves [SqlMatchRecognize#getAfter()] as `null`, and `SqlToRelConverter` then silently +/// substitutes [SqlMatchRecognize.AfterOption#SKIP_TO_NEXT_ROW]. SQL:2016 (and Trino, Snowflake and Oracle) all +/// default to `SKIP PAST LAST ROW`. The two differ in whether matches may overlap, so a query ported from +/// another engine would silently return different rows. Conversion erases the omitted-vs-explicit distinction, +/// so the fix has to happen here. See [#applyStandardAfterMatchDefault]. +/// 2. **Unsupported constructs must be rejected before conversion.** Several of them (`ORDER BY` / `PARTITION BY` +/// on arbitrary expressions in particular) blow up inside `SqlToRelConverter` with an `AssertionError` or +/// `ClassCastException` rather than a usable message. +/// +/// Everything this class rejects is deferred, not permanently unsupported; each message names the construct so the +/// user can rewrite the query. +/// +/// This class is stateless apart from the tree it is walking and is **not** thread-safe; create a new instance per +/// query, exactly like [RowExpressionValidationVisitor]. +public class MatchRecognizeValidator extends SqlBasicVisitor { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchRecognizeValidator.class); + + private static final String PERMUTE = "PERMUTE"; + private static final String PERMUTE_NOT_SUPPORTED = + "PERMUTE is not supported yet in the MATCH_RECOGNIZE PATTERN clause. Expand the permutation into an explicit " + + "alternation, e.g. PATTERN ((A B) | (B A))."; + + /// Whether the node currently being visited sits inside a MEASURES or DEFINE list, the only two places where the + /// `RUNNING` / `FINAL` semantics modifiers are legal. + private boolean _inMeasureOrDefine; + private boolean _matchRecognizeFound; + + /// Whether the visited SQL tree contains at least one MATCH_RECOGNIZE clause. + public boolean hasMatchRecognize() { + return _matchRecognizeFound; + } + + @Override + public Void visit(SqlCall call) { + if (call instanceof SqlMatchRecognize) { + visitMatchRecognize((SqlMatchRecognize) call); + return null; + } + SqlKind kind = call.getKind(); + if ((kind == SqlKind.RUNNING || kind == SqlKind.FINAL) && !_inMeasureOrDefine) { + // The RUNNING / FINAL prefix operators are registered for MATCH_RECOGNIZE, but unlike PREV / NEXT / FIRST / + // LAST / CLASSIFIER / MATCH_NUMBER, Calcite does not confine them to a MatchRecognizeScope. Reject them here + // so they cannot leak into an ordinary projection. + throw unsupported(kind + " can only be used in the MEASURES or DEFINE clause of a MATCH_RECOGNIZE query: '" + + call + "'."); + } + return super.visit(call); + } + + private void visitMatchRecognize(SqlMatchRecognize match) { + _matchRecognizeFound = true; + validateAndRewrite(match); + // Only MEASURES and DEFINE may contain RUNNING / FINAL. Every other operand has already been checked by + // validateAndRewrite(), so the only one left to descend into is the table reference, which may itself hold a + // nested MATCH_RECOGNIZE. + visitIn(match.getMeasureList(), true); + visitIn(match.getPatternDefList(), true); + visitIn(match.getTableRef(), false); + } + + private void visitIn(SqlNode node, boolean inMeasureOrDefine) { + boolean saved = _inMeasureOrDefine; + _inMeasureOrDefine = inMeasureOrDefine; + try { + node.accept(this); + } finally { + _inMeasureOrDefine = saved; + } + } + + private void validateAndRewrite(SqlMatchRecognize match) { + rejectUnsupportedClauses(match); + validatePartitionBy(match.getPartitionList()); + validateOrderBy(match.getOrderList()); + Set definedVariables = definedVariables(match.getPatternDefList()); + validatePattern(match.getPattern(), definedVariables); + validateDefinitions(match.getPatternDefList()); + // Deliberately last: a query that also uses a deferred construct should be told about that construct first. + validateMeasures(match); + rejectRowSourceAliasCollision(match, definedVariables); + applyStandardAfterMatchDefault(match); + warnOnUndefinedPatternVariables(match, definedVariables); + } + + /// Rewrites an omitted `AFTER MATCH` clause into an explicit `AFTER MATCH SKIP PAST LAST ROW`. + /// + /// Calcite's `SqlToRelConverter` defaults a `null` `AFTER` operand to `SKIP TO NEXT ROW`, which produces + /// overlapping matches. SQL:2016 mandates `SKIP PAST LAST ROW` (non-overlapping matches), and that is what + /// Trino, Snowflake and Oracle do. An explicitly written `AFTER MATCH SKIP TO NEXT ROW` is left untouched: the + /// parser only leaves the operand `null` when the clause is absent from the query text, so the omitted and the + /// explicit case are perfectly distinguishable here. + private void applyStandardAfterMatchDefault(SqlMatchRecognize match) { + if (match.getAfter() != null) { + return; + } + match.setOperand(SqlMatchRecognize.OPERAND_AFTER, + SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW.symbol(match.getParserPosition())); + } + + private void rejectUnsupportedClauses(SqlMatchRecognize match) { + SqlLiteral rowsPerMatch = match.getRowsPerMatch(); + if (rowsPerMatch != null + && rowsPerMatch.getValue() == SqlMatchRecognize.RowsPerMatchOption.ALL_ROWS) { + throw unsupported("ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the " + + "default) and expose the per-match values you need through the MEASURES clause."); + } + if (!isEmpty(match.getSubsetList())) { + throw unsupported("SUBSET is not supported yet in MATCH_RECOGNIZE. Remove the SUBSET clause and reference the " + + "individual pattern variables directly."); + } + if (match.getInterval() != null) { + throw unsupported("WITHIN is not supported yet in MATCH_RECOGNIZE. Remove the WITHIN clause and bound the " + + "match with a predicate in the DEFINE clause instead."); + } + } + + /// Rejects the two `MEASURES` shapes whose Calcite row type is not `partition columns ++ measures`, which is what + /// `MatchNode` and `MatchOperator` assume. Both plan without complaint today and only fail on the server, with a + /// message describing an internal invariant rather than the user's mistake. + /// + /// 1. **No `MEASURES` clause at all.** `SqlValidatorImpl.validateMatchRecognize` ends with + /// `if (measureList.size() == 0) ns.setType()`, i.e. the whole *input* row type, + /// which is not the `ONE ROW PER MATCH` shape at all. `MEASURES` is optional in SQL:2016 and in Trino, so this + /// is ordinary valid SQL; we do not own the Calcite code, so rejecting is the only sound option until + /// measures-less `MATCH_RECOGNIZE` is implemented properly. + /// 2. **An alias that collides with a `PARTITION BY` column or with an earlier measure alias.** Calcite adds each + /// measure to the row type only `if (!typeBuilder.nameExists(name))`, so a colliding alias is dropped + /// altogether and the query plans with a column missing. + /// + /// The name comparison is case-sensitive on purpose, because `RelDataTypeFactory.Builder#nameExists` is: + /// `PARTITION BY col1 ... MEASURES LAST(x) AS COL1` really does produce two distinct columns and must keep working. + private void validateMeasures(SqlMatchRecognize match) { + SqlNodeList measureList = match.getMeasureList(); + if (isEmpty(measureList)) { + throw unsupported("MATCH_RECOGNIZE requires a MEASURES clause. ONE ROW PER MATCH emits the PARTITION BY " + + "columns plus the MEASURES, so add at least one measure, e.g. MEASURES MATCH_NUMBER() AS mno."); + } + Set names = new LinkedHashSet<>(); + for (SqlNode partitionKey : match.getPartitionList()) { + // validatePartitionBy() already established that every partition key is a plain identifier. + names.add(lastName((SqlIdentifier) partitionKey)); + } + for (SqlNode measure : measureList) { + SqlNode alias = asOperand(measure, 1); + if (alias instanceof SqlIdentifier && ((SqlIdentifier) alias).isSimple() + && !names.add(((SqlIdentifier) alias).getSimple())) { + throw unsupported("MATCH_RECOGNIZE measure alias '" + ((SqlIdentifier) alias).getSimple() + "' collides with " + + "a PARTITION BY column or an earlier measure alias. Calcite drops the duplicate from the output row " + + "type, which would silently lose a column; rename the measure."); + } + } + } + + /// Rejects a pattern variable whose name equals the row source alias. + /// + /// Calcite has no separate representation for the SQL:2016 universal row pattern variable: it qualifies an + /// unqualified column reference in `MEASURES` / `DEFINE` with the row source alias, and + /// `RelToPlanNodeConverter#bindPatternFieldRefs` recognises the universal variable purely by that alpha not being a + /// pattern variable name. When the two names coincide, `LAST(col)` silently becomes "last row mapped to that + /// variable" instead of "last row of the match", and an unqualified reference in a `DEFINE` predicate changes which + /// rows match at all. Nothing is left at the `RexNode` level to disambiguate, so it has to be caught here. + /// + /// The comparison is case-sensitive because the symbol table is a plain `Map` lookup: + /// `FROM a AS aa ... PATTERN (AA B+)` binds to the universal variable correctly today and must keep working. + private void rejectRowSourceAliasCollision(SqlMatchRecognize match, Set definedVariables) { + String alias = SqlValidatorUtil.alias(match.getTableRef()); + if (alias == null) { + // A join or a sub-query with no derivable alias; Calcite does not qualify anything with a row source alias. + return; + } + Set variables = new LinkedHashSet<>(definedVariables); + collectPatternVariables(match.getPattern(), variables); + if (variables.contains(alias)) { + throw unsupported("MATCH_RECOGNIZE pattern variable '" + alias + "' collides with the row source alias '" + + alias + "'. Calcite qualifies unqualified column references in MEASURES and DEFINE with the row source " + + "alias, so they would silently bind to that pattern variable instead of to the universal row pattern " + + "variable. Rename the pattern variable, or alias the input table differently."); + } + } + + private static String lastName(SqlIdentifier identifier) { + return identifier.names.get(identifier.names.size() - 1); + } + + private void validatePartitionBy(SqlNodeList partitionList) { + for (SqlNode partitionKey : partitionList) { + if (!(partitionKey instanceof SqlIdentifier)) { + throw unsupported("PARTITION BY on expressions is not supported yet in MATCH_RECOGNIZE: '" + partitionKey + + "'. Only plain column references are supported. Compute the expression in a sub-query and partition " + + "by the resulting column."); + } + } + } + + private void validateOrderBy(SqlNodeList orderList) { + if (isEmpty(orderList)) { + throw unsupported("MATCH_RECOGNIZE requires an ORDER BY clause. Row pattern recognition is only well defined " + + "over an ordered sequence of rows, so add ORDER BY inside the MATCH_RECOGNIZE clause."); + } + for (SqlNode orderKey : orderList) { + SqlNode key = orderKey; + SqlKind kind = key.getKind(); + if (kind == SqlKind.NULLS_FIRST || kind == SqlKind.NULLS_LAST) { + throw unsupported("NULLS FIRST / NULLS LAST is not supported yet in the MATCH_RECOGNIZE ORDER BY clause: '" + + orderKey + "'. Remove the null ordering, or filter the null values out before the MATCH_RECOGNIZE " + + "clause."); + } + if (kind == SqlKind.DESCENDING) { + key = ((SqlCall) key).operand(0); + } + if (!(key instanceof SqlIdentifier)) { + throw unsupported("ORDER BY on expressions is not supported yet in MATCH_RECOGNIZE: '" + orderKey + "'. Only " + + "plain column references, optionally followed by ASC or DESC, are supported. Compute the expression in " + + "a sub-query and order by the resulting column."); + } + } + } + + private void validatePattern(SqlNode pattern, Set definedVariables) { + if (pattern instanceof SqlIdentifier) { + SqlIdentifier identifier = (SqlIdentifier) pattern; + // PERMUTE is a non-reserved keyword, so `PATTERN (PERMUTE(A))` parses as the concatenation of a pattern + // variable named PERMUTE and the group (A) instead of as a permutation. Rejecting an undefined PERMUTE + // variable turns that silently wrong plan into an explicit error. A variable that is actually DEFINEd is a + // legitimate (if unfortunate) name and is left alone. + if (identifier.isSimple() && PERMUTE.equalsIgnoreCase(identifier.getSimple()) + && !definedVariables.contains(identifier.getSimple())) { + throw unsupported(PERMUTE_NOT_SUPPORTED); + } + return; + } + if (!(pattern instanceof SqlCall)) { + return; + } + SqlCall call = (SqlCall) pattern; + SqlKind kind = call.getKind(); + if (kind == SqlKind.PATTERN_PERMUTE) { + throw unsupported(PERMUTE_NOT_SUPPORTED); + } + if (kind == SqlKind.PATTERN_EXCLUDED) { + throw unsupported("Pattern exclusions '{- -}' are not supported yet in the MATCH_RECOGNIZE PATTERN clause. " + + "Remove the exclusion; note that exclusions only affect ALL ROWS PER MATCH output, which is also not " + + "supported yet."); + } + for (SqlNode operand : call.getOperandList()) { + if (operand != null) { + validatePattern(operand, definedVariables); + } + } + } + + /// Rejects aggregate calls inside `DEFINE`. Aggregates are legal in SQL:2016 `DEFINE` (they aggregate over the + /// rows matched so far), but Pinot does not implement running aggregation inside the pattern matcher yet. + private void validateDefinitions(SqlNodeList patternDefList) { + for (SqlNode definition : patternDefList) { + SqlNode condition = patternDefinitionCondition(definition); + if (condition != null) { + rejectAggregates(condition, definition); + } + } + } + + private void rejectAggregates(SqlNode node, SqlNode definition) { + if (!(node instanceof SqlCall)) { + return; + } + SqlCall call = (SqlCall) node; + if (isAggregate(call.getOperator())) { + throw unsupported("Aggregate function '" + call.getOperator().getName() + "' is not supported yet in the " + + "MATCH_RECOGNIZE DEFINE clause: '" + definition + "'. Only row level predicates, optionally using " + + "PREV / NEXT / FIRST / LAST / CLASSIFIER / MATCH_NUMBER, are supported."); + } + for (SqlNode operand : call.getOperandList()) { + if (operand != null) { + rejectAggregates(operand, definition); + } + } + } + + /// Warns about pattern variables that appear in `PATTERN` but are never `DEFINE`d. SQL:2016 says such a variable + /// matches every row (its condition defaults to `TRUE`), so this is legal, but it is far more often a typo. + private void warnOnUndefinedPatternVariables(SqlMatchRecognize match, Set definedVariables) { + Set undefinedVariables = new LinkedHashSet<>(); + collectPatternVariables(match.getPattern(), undefinedVariables); + undefinedVariables.removeAll(definedVariables); + if (!undefinedVariables.isEmpty()) { + LOGGER.warn("MATCH_RECOGNIZE pattern variable(s) {} are used in PATTERN but never DEFINEd. Per SQL:2016 they " + + "match every row (their condition defaults to TRUE), which is usually an unintended typo. Query pattern: " + + "{}", undefinedVariables, match.getPattern()); + } + } + + private static Set definedVariables(SqlNodeList patternDefList) { + Set definedVariables = new LinkedHashSet<>(); + for (SqlNode definition : patternDefList) { + SqlIdentifier variable = patternDefinitionVariable(definition); + if (variable != null && variable.isSimple()) { + definedVariables.add(variable.getSimple()); + } + } + return definedVariables; + } + + private void collectPatternVariables(SqlNode pattern, Set variables) { + if (pattern instanceof SqlIdentifier) { + SqlIdentifier identifier = (SqlIdentifier) pattern; + if (identifier.isSimple()) { + variables.add(identifier.getSimple()); + } + return; + } + if (!(pattern instanceof SqlCall)) { + // Quantifier bounds and the reluctant flag are literals; they are not pattern variables. + return; + } + for (SqlNode operand : ((SqlCall) pattern).getOperandList()) { + if (operand != null) { + collectPatternVariables(operand, variables); + } + } + } + + /// Returns the condition of a `DEFINE` item. The parser represents `DEFINE var AS condition` as + /// `AS(condition, var)`. + @Nullable + private static SqlNode patternDefinitionCondition(SqlNode definition) { + return asOperand(definition, 0); + } + + /// Returns the variable of a `DEFINE` item. The parser represents `DEFINE var AS condition` as `AS(condition, var)`. + @Nullable + private static SqlIdentifier patternDefinitionVariable(SqlNode definition) { + SqlNode variable = asOperand(definition, 1); + return variable instanceof SqlIdentifier ? (SqlIdentifier) variable : null; + } + + @Nullable + private static SqlNode asOperand(SqlNode definition, int index) { + if (definition.getKind() != SqlKind.AS) { + return null; + } + List operands = ((SqlCall) definition).getOperandList(); + return operands.size() > index ? operands.get(index) : null; + } + + /// The parser half-resolves calls against [org.apache.calcite.sql.fun.SqlStdOperatorTable], so standard aggregates + /// such as `SUM` and `COUNT` already carry a [SqlAggFunction] here. Pinot specific aggregates (`DISTINCTCOUNT`, + /// `PERCENTILE...`, ...) are still [SqlUnresolvedFunction] at this point and are matched by name instead. + private static boolean isAggregate(SqlOperator operator) { + if (operator instanceof SqlAggFunction) { + return true; + } + return operator instanceof SqlUnresolvedFunction + && AggregationFunctionType.isAggregationFunction(operator.getName()); + } + + private static boolean isEmpty(@Nullable SqlNodeList nodeList) { + return nodeList == null || nodeList.isEmpty(); + } + + private static UnsupportedMatchRecognizeException unsupported(String message) { + return new UnsupportedMatchRecognizeException(message); + } + + /// Thrown when a `MATCH_RECOGNIZE` query uses a construct that Pinot does not support yet. The message always names + /// the construct and suggests a rewrite. + public static class UnsupportedMatchRecognizeException extends RuntimeException { + public UnsupportedMatchRecognizeException(String message) { + super(message); + } + } +} diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRuleTest.java b/pinot-query-planner/src/test/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRuleTest.java new file mode 100644 index 000000000000..12e1389ea8c9 --- /dev/null +++ b/pinot-query-planner/src/test/java/org/apache/pinot/calcite/rel/rules/PinotMatchExchangeNodeInsertRuleTest.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.calcite.rel.rules; + +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelDistributions; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Match; +import org.apache.calcite.rel.logical.LogicalMatch; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange; +import org.apache.pinot.calcite.rel.logical.PinotLogicalSortExchange; +import org.apache.pinot.query.QueryEnvironment; +import org.apache.pinot.query.context.PlannerContext; +import org.apache.pinot.query.type.TypeFactory; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.exception.QueryException; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Unit tests for [PinotMatchExchangeNodeInsertRule], which inserts the exchange below MATCH_RECOGNIZE. +/// +/// The rule is driven directly instead of through a full query so that a [Match] without ORDER BY - which the +/// SQL front end rejects - can still be covered. Plan-shape assertions for real queries live in +/// `MatchRecognizeExchangePlanTest`. +public class PinotMatchExchangeNodeInsertRuleTest { + private static final TypeFactory TYPE_FACTORY = new TypeFactory(); + private static final RexBuilder REX_BUILDER = new RexBuilder(TYPE_FACTORY); + // (col1 VARCHAR, col2 VARCHAR, ts BIGINT) + private static final RelDataType ROW_TYPE = TYPE_FACTORY.builder() + .add("col1", TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR)) + .add("col2", TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR)) + .add("ts", TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)) + .build(); + + @Test + public void testPartitionKeysArePrependedToCollation() { + // PARTITION BY col1 ORDER BY ts + Match match = createMatch(ImmutableBitSet.of(0), RelCollations.of(2)); + PinotLogicalSortExchange exchange = (PinotLogicalSortExchange) applyRule(match, Map.of()).getInput(0); + + assertEquals(exchange.getDistribution().getKeys(), List.of(0)); + assertEquals(exchange.getCollation().getKeys(), List.of(0, 2)); + assertFalse(exchange.isSortOnSender()); + assertTrue(exchange.isSortOnReceiver()); + } + + @Test + public void testAllPartitionKeysArePrependedInOrder() { + // PARTITION BY col1, col2 ORDER BY ts DESC + Match match = createMatch(ImmutableBitSet.of(0, 1), + RelCollations.of(new RelFieldCollation(2, RelFieldCollation.Direction.DESCENDING))); + PinotLogicalSortExchange exchange = (PinotLogicalSortExchange) applyRule(match, Map.of()).getInput(0); + + assertEquals(exchange.getDistribution().getType(), RelDistribution.Type.HASH_DISTRIBUTED); + assertEquals(exchange.getDistribution().getKeys(), List.of(0, 1)); + List fieldCollations = exchange.getCollation().getFieldCollations(); + assertEquals(fieldCollations.size(), 3); + assertEquals(fieldCollations.get(0).getFieldIndex(), 0); + assertEquals(fieldCollations.get(0).getDirection(), RelFieldCollation.Direction.ASCENDING); + assertEquals(fieldCollations.get(1).getFieldIndex(), 1); + assertEquals(fieldCollations.get(1).getDirection(), RelFieldCollation.Direction.ASCENDING); + // The ORDER BY key keeps its own direction. + assertEquals(fieldCollations.get(2).getFieldIndex(), 2); + assertEquals(fieldCollations.get(2).getDirection(), RelFieldCollation.Direction.DESCENDING); + } + + @Test + public void testOrderKeyThatIsAlsoPartitionKeyIsNotRepeated() { + // PARTITION BY col1 ORDER BY col1 DESC, ts: col1 is constant within a partition, so it must not appear twice. + Match match = createMatch(ImmutableBitSet.of(0), + RelCollations.of(new RelFieldCollation(0, RelFieldCollation.Direction.DESCENDING), + new RelFieldCollation(2))); + PinotLogicalSortExchange exchange = (PinotLogicalSortExchange) applyRule(match, Map.of()).getInput(0); + + assertEquals(exchange.getCollation().getKeys(), List.of(0, 2)); + assertEquals(exchange.getCollation().getFieldCollations().get(0).getDirection(), + RelFieldCollation.Direction.ASCENDING); + } + + @Test + public void testMissingPartitionByIsRejected() { + Match match = createMatch(ImmutableBitSet.of(), RelCollations.of(2)); + + QueryException exception = expectThrows(QueryException.class, () -> applyRule(match, Map.of())); + assertEquals(exception.getErrorCode(), QueryErrorCode.QUERY_PLANNING); + assertTrue(exception.getMessage().contains("MATCH_RECOGNIZE without a PARTITION BY clause is not allowed"), + exception.getMessage()); + // The message has to name the escape hatch, otherwise the user cannot act on it. + assertTrue(exception.getMessage().contains(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY), + exception.getMessage()); + } + + @Test + public void testMissingPartitionByIsAllowedByQueryOption() { + Match match = createMatch(ImmutableBitSet.of(), RelCollations.of(2)); + Map options = Map.of(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY, "true"); + PinotLogicalSortExchange exchange = (PinotLogicalSortExchange) applyRule(match, options).getInput(0); + + // Hashing on zero keys funnels every row to a single worker, which is exactly what the option opts into. + assertTrue(exchange.getDistribution().getKeys().isEmpty()); + assertEquals(exchange.getCollation().getKeys(), List.of(2)); + } + + @Test + public void testQueryOptionValueOtherThanTrueStillRejects() { + Match match = createMatch(ImmutableBitSet.of(), RelCollations.of(2)); + Map options = Map.of(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY, "false"); + + expectThrows(QueryException.class, () -> applyRule(match, options)); + } + + @Test + public void testNoPartitionByAndNoOrderByUsesPlainExchange() { + // Degenerate case: nothing to sort on, so a sort exchange would be pure overhead. + Match match = createMatch(ImmutableBitSet.of(), RelCollations.EMPTY); + Map options = Map.of(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY, "true"); + RelNode input = applyRule(match, options).getInput(0); + + assertTrue(input instanceof PinotLogicalExchange, input.getClass().getName()); + assertTrue(((PinotLogicalExchange) input).getDistribution().getKeys().isEmpty()); + } + + @Test + public void testPartitionByWithoutOrderByStillClustersByPartition() { + Match match = createMatch(ImmutableBitSet.of(0, 1), RelCollations.EMPTY); + PinotLogicalSortExchange exchange = (PinotLogicalSortExchange) applyRule(match, Map.of()).getInput(0); + + assertEquals(exchange.getDistribution().getKeys(), List.of(0, 1)); + assertEquals(exchange.getCollation().getKeys(), List.of(0, 1)); + } + + @Test + public void testMatchPropertiesArePreserved() { + Match match = createMatch(ImmutableBitSet.of(0), RelCollations.of(2)); + Match transformed = applyRule(match, Map.of()); + + assertSame(transformed.getPattern(), match.getPattern()); + assertSame(transformed.getAfter(), match.getAfter()); + assertEquals(transformed.getPatternDefinitions(), match.getPatternDefinitions()); + assertEquals(transformed.getMeasures(), match.getMeasures()); + assertEquals(transformed.getPartitionKeys(), match.getPartitionKeys()); + assertEquals(transformed.getOrderKeys(), match.getOrderKeys()); + assertEquals(transformed.getRowType(), match.getRowType()); + } + + @Test + public void testRuleDoesNotFireWhenInputIsAlreadyAnExchange() { + Match match = createMatch(ImmutableBitSet.of(0), RelCollations.of(2)); + Match matchOverExchange = (Match) match.copy(match.getTraitSet(), + List.of(PinotLogicalExchange.create(match.getInput(), RelDistributions.hash(List.of(0))))); + + RelOptRuleCall call = Mockito.mock(RelOptRuleCall.class); + Mockito.when(call.rel(0)).thenReturn(matchOverExchange); + assertFalse(PinotMatchExchangeNodeInsertRule.INSTANCE.matches(call)); + // ... but it does fire when the input is not an exchange yet. + Mockito.when(call.rel(0)).thenReturn(match); + assertTrue(PinotMatchExchangeNodeInsertRule.INSTANCE.matches(call)); + } + + /// Runs the rule on the given match and returns the rewritten [Match]. The planner context carries the query + /// options so the rule can read them the same way it does during real planning. + private static Match applyRule(Match match, Map options) { + PlannerContext plannerContext = + PlannerContext.forTesting(options, Mockito.mock(QueryEnvironment.Config.class)); + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build(), plannerContext); + RelOptRuleCall call = Mockito.mock(RelOptRuleCall.class); + Mockito.when(call.rel(0)).thenReturn(match); + Mockito.when(call.getPlanner()).thenReturn(planner); + + PinotMatchExchangeNodeInsertRule.INSTANCE.onMatch(call); + + ArgumentCaptor captor = ArgumentCaptor.forClass(RelNode.class); + Mockito.verify(call, Mockito.times(1)).transformTo(captor.capture()); + return (Match) captor.getValue(); + } + + private static Match createMatch(ImmutableBitSet partitionKeys, RelCollation orderKeys) { + HepPlanner planner = new HepPlanner(new HepProgramBuilder().build()); + RelOptCluster cluster = RelOptCluster.create(planner, REX_BUILDER); + RelNode input = LogicalValues.createEmpty(cluster, ROW_TYPE); + RexNode pattern = REX_BUILDER.makeLiteral("A"); + RexNode after = REX_BUILDER.makeFlag(SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW); + Map patternDefinitions = Map.of("A", REX_BUILDER.makeLiteral(true)); + Map measures = Map.of(); + Map> subsets = Map.of(); + return LogicalMatch.create(input, ROW_TYPE, pattern, false, false, patternDefinitions, measures, after, subsets, + false, partitionKeys, orderKeys, null); + } +} diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/query/QueryEnvironmentTestBase.java b/pinot-query-planner/src/test/java/org/apache/pinot/query/QueryEnvironmentTestBase.java index 968b9b0b5f41..297ede73f80f 100644 --- a/pinot-query-planner/src/test/java/org/apache/pinot/query/QueryEnvironmentTestBase.java +++ b/pinot-query-planner/src/test/java/org/apache/pinot/query/QueryEnvironmentTestBase.java @@ -304,6 +304,14 @@ protected Object[][] provideQueries() { public static QueryEnvironment getQueryEnvironment(int reducerPort, int port1, int port2, Map schemaMap, Map> segmentMap1, Map> segmentMap2, @Nullable Map>>> partitionedSegmentsMap) { + return getQueryEnvironment(reducerPort, port1, port2, schemaMap, segmentMap1, segmentMap2, + partitionedSegmentsMap, CommonConstants.Broker.DEFAULT_USE_PHYSICAL_OPTIMIZER); + } + + public static QueryEnvironment getQueryEnvironment(int reducerPort, int port1, int port2, + Map schemaMap, Map> segmentMap1, Map> segmentMap2, + @Nullable Map>>> partitionedSegmentsMap, + boolean defaultUsePhysicalOptimizer) { MockRoutingManagerFactory factory = new MockRoutingManagerFactory(port1, port2); for (Map.Entry entry : schemaMap.entrySet()) { factory.registerTable(entry.getValue(), entry.getKey()); @@ -341,8 +349,14 @@ public static QueryEnvironment getQueryEnvironment(int reducerPort, int port1, i } RoutingManager routingManager = factory.buildRoutingManager(partitionInfoMap); TableCache tableCache = factory.buildTableCache(); - return new QueryEnvironment(CommonConstants.DEFAULT_DATABASE, tableCache, - new WorkerManager("Broker_localhost", "localhost", reducerPort, routingManager)); + return new QueryEnvironment(QueryEnvironment.configBuilder() + .requestId(-1L) + .database(CommonConstants.DEFAULT_DATABASE) + .tableCache(tableCache) + .workerManager(new WorkerManager("Broker_localhost", "localhost", reducerPort, routingManager)) + .isNullHandlingEnabled(true) + .defaultUsePhysicalOptimizer(defaultUsePhysicalOptimizer) + .build()); } /// JSON test case definition for query planner test cases. Tables and schemas will come from those already defined diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/MatchNodeConverterTest.java b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/MatchNodeConverterTest.java new file mode 100644 index 000000000000..481f545b179f --- /dev/null +++ b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/MatchNodeConverterTest.java @@ -0,0 +1,443 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.logical; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Match; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; +import org.apache.pinot.calcite.sql.fun.PinotOperatorTable; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; +import org.apache.pinot.query.QueryEnvironmentTestBase; +import org.apache.pinot.query.planner.physical.DispatchablePlanFragment; +import org.apache.pinot.query.planner.physical.DispatchableSubPlan; +import org.apache.pinot.query.planner.plannode.ExplainedNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.planner.plannode.PlanNode; +import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; +import org.apache.pinot.query.planner.plannode.RowPattern; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.apache.pinot.sql.parsers.CalciteSqlParser; +import org.apache.pinot.sql.parsers.PinotSqlType; +import org.apache.pinot.sql.parsers.SqlNodeAndOptions; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Tests the two MATCH_RECOGNIZE rel conversions: [RelToPlanNodeConverter] lowering Calcite's `LogicalMatch` into a +/// [MatchNode], and [PlanNodeToRelConverter] rebuilding a `LogicalMatch` from it. +public class MatchNodeConverterTest extends QueryEnvironmentTestBase { + + @Test(dataProvider = "patternQueries") + public void testPatternIsLowered(String patternClause, String expectedPattern) { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (" + patternClause + ") DEFINE A AS A.col3 > 0, B AS B.col3 > 0, C AS C.col3 > 0)"); + assertEquals(match.getPatternString(), expectedPattern); + } + + @DataProvider(name = "patternQueries") + public Object[][] patternQueries() { + //@formatter:off + return new Object[][]{ + new Object[]{"A", "A"}, + // Calcite builds a left deep binary PATTERN_CONCAT tree; it is flattened into a single n-ary node. + new Object[]{"A B C", "A B C"}, + new Object[]{"A | B | C", "(A | B | C)"}, + new Object[]{"A (B | C)", "A (B | C)"}, + new Object[]{"A*", "A*"}, + new Object[]{"A+", "A+"}, + new Object[]{"A?", "A?"}, + new Object[]{"A{3}", "A{3}"}, + new Object[]{"A{2,}", "A{2,}"}, + new Object[]{"A{2,5}", "A{2,5}"}, + // Calcite writes -1 for the omitted minimum of `{,m}`; SQL:2016 reads it as 0. + new Object[]{"A{,4}", "A{0,4}"}, + new Object[]{"A*?", "A*?"}, + new Object[]{"A+?", "A+?"}, + new Object[]{"A{2,5}?", "A{2,5}?"}, + // A quantifier over a concatenation has to be parenthesized when rendered back. + new Object[]{"(A B)+", "(A B)+"}, + new Object[]{"^ A B $", "^ A B $"}, + new Object[]{"^ A", "^ A"}, + new Object[]{"A $", "A $"}, + new Object[]{"^ (A | B) $", "^ (A | B) $"}, + new Object[]{"^ A B{2,3} (B | C)+? $", "^ A B{2,3} (B | C)+? $"} + }; + //@formatter:on + } + + @Test + public void testQuantifierGreediness() { + RowPattern greedy = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (A*) DEFINE A AS A.col3 > 0)").getPattern(); + RowPattern reluctant = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (A*?) DEFINE A AS A.col3 > 0)").getPattern(); + assertTrue(((RowPattern.Quantifier) greedy).isGreedy()); + assertFalse(((RowPattern.Quantifier) reluctant).isGreedy()); + assertEquals(((RowPattern.Quantifier) greedy).getMinRepeat(), 0); + assertEquals(((RowPattern.Quantifier) greedy).getMaxRepeat(), RowPattern.Quantifier.UNBOUNDED); + } + + @Test + public void testSymbolTableAndDefinitions() { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (A B C) DEFINE C AS C.col3 > 0, A AS A.col3 > 0)"); + // Ordered by first appearance in PATTERN, not by DEFINE order, so the ordinals do not depend on the iteration + // order of Calcite's pattern definition map. B has no DEFINE entry, so it matches every row. + assertEquals(symbolNames(match), List.of("A", "B", "C")); + assertNotNull(match.getPatternSymbols().get(0).getDefinition()); + assertNull(match.getPatternSymbols().get(1).getDefinition()); + assertNotNull(match.getPatternSymbols().get(2).getDefinition()); + } + + @Test + public void testPatternFieldRefsAreBoundToSymbolOrdinals() { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts " + + "MEASURES LAST(B.col3) AS ev PATTERN (A B) DEFINE A AS A.col3 > 0, B AS B.col3 > A.col3)"); + assertEquals(symbolNames(match), List.of("A", "B")); + // DEFINE B references both B and A; every reference must carry the ordinal of its variable. + List refs = collectPatternFieldRefs(match.getPatternSymbols().get(1) + .getDefinition()); + assertEquals(refs.size(), 2); + assertEquals(refs.get(0).getAlpha(), "B"); + assertEquals(refs.get(0).getSymbolOrdinal(), 1); + assertEquals(refs.get(1).getAlpha(), "A"); + assertEquals(refs.get(1).getSymbolOrdinal(), 0); + // MEASURES references are bound too. + List measureRefs = + collectPatternFieldRefs(match.getMeasures().get(0).getExpression()); + assertEquals(measureRefs.size(), 1); + assertEquals(measureRefs.get(0).getSymbolOrdinal(), 1); + } + + @Test + public void testUnqualifiedReferenceBindsToUniversalSymbol() { + // `col3` is not qualified by a pattern variable. Calcite has no representation for the SQL:2016 universal row + // pattern variable and reuses the row source alias ("a") as the alpha, so it must not be mistaken for a variable. + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (A) DEFINE A AS col3 > 0)"); + assertEquals(symbolNames(match), List.of("A")); + List refs = + collectPatternFieldRefs(match.getPatternSymbols().get(0).getDefinition()); + assertEquals(refs.size(), 1); + assertEquals(refs.get(0).getAlpha(), "a"); + assertEquals(refs.get(0).getSymbolOrdinal(), RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL); + } + + @Test(dataProvider = "afterMatchQueries") + public void testAfterMatchSkipMode(String afterClause, MatchNode.AfterMatchSkipMode expectedMode, + int expectedOrdinal) { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + afterClause + " PATTERN (A B) DEFINE A AS A.col3 > 0, B AS B.col3 > 0)"); + assertEquals(match.getAfterMatchSkipMode(), expectedMode); + assertEquals(match.getAfterMatchSkipToSymbolOrdinal(), expectedOrdinal); + } + + @DataProvider(name = "afterMatchQueries") + public Object[][] afterMatchQueries() { + //@formatter:off + return new Object[][]{ + // An omitted AFTER MATCH must reach the plan as SKIP PAST LAST ROW (SQL:2016), not Calcite's SKIP TO NEXT ROW. + new Object[]{"", MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL}, + new Object[]{ + "AFTER MATCH SKIP PAST LAST ROW", MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL + }, + new Object[]{ + "AFTER MATCH SKIP TO NEXT ROW", MatchNode.AfterMatchSkipMode.TO_NEXT_ROW, MatchNode.NO_SKIP_TO_SYMBOL + }, + // Ordinal 0 is a legal target and must stay distinguishable from "no target". + new Object[]{"AFTER MATCH SKIP TO FIRST A", MatchNode.AfterMatchSkipMode.TO_FIRST, 0}, + new Object[]{"AFTER MATCH SKIP TO LAST B", MatchNode.AfterMatchSkipMode.TO_LAST, 1} + }; + //@formatter:on + } + + @Test + public void testPartitionOrderAndMeasures() { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts DESC " + + "MEASURES MATCH_NUMBER() AS mno, CLASSIFIER() AS cls PATTERN (A) DEFINE A AS A.col3 > 0)"); + assertEquals(match.getPartitionKeys().size(), 1); + assertEquals(match.getCollations().size(), 1); + assertEquals(match.getRowsPerMatchMode(), MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + assertEquals(match.getMeasures().size(), 2); + assertEquals(match.getMeasures().get(0).getName(), "mno"); + assertEquals(match.getMeasures().get(1).getName(), "cls"); + assertEquals(match.explain(), "MATCH_RECOGNIZE"); + } + + /// Calcite's `Match#getPartitionKeys()` is an `ImmutableBitSet`, so `asList()` is ascending index order and the + /// PARTITION BY source order survives only in the output row type. `MatchOperator` fills the output row + /// positionally, so passing the bitset order through would report each partition column's value under the other + /// one's name - or, as in the first case below where the two types differ, blow up in `TypeUtils.convertRow`. + @Test(dataProvider = "partitionKeyOrderQueries") + public void testPartitionKeysKeepTheirSourceOrder(String partitionBy, List expectedKeys, + String[] expectedNames, ColumnDataType[] expectedTypes) { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY " + partitionBy + " ORDER BY ts " + + "MEASURES LAST(X.col3) AS lastv PATTERN (X+) DEFINE X AS X.col3 > 0)"); + assertEquals(match.getPartitionKeys(), expectedKeys); + assertEquals(match.getDataSchema().getColumnNames(), expectedNames); + assertEquals(match.getDataSchema().getColumnDataTypes(), expectedTypes); + } + + @DataProvider(name = "partitionKeyOrderQueries") + public Object[][] partitionKeyOrderQueries() { + // Input columns of table `a`: col1 STRING = 0, col2 STRING = 1, col3 INT = 2. + //@formatter:off + return new Object[][]{ + // Descending index order: this is what the ImmutableBitSet loses. The two columns have different types, so a + // regression shows up as a type error rather than only as a value swap. + new Object[]{ + "col3, col1", List.of(2, 0), new String[]{"col3", "col1", "lastv"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.STRING, ColumnDataType.INT} + }, + // Two same-typed columns in descending index order: a regression here is a silent value swap. + new Object[]{ + "col2, col1", List.of(1, 0), new String[]{"col2", "col1", "lastv"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.STRING, ColumnDataType.INT} + }, + // The already-ascending case, which was correct by accident before and must stay correct. + new Object[]{ + "col1, col3", List.of(0, 2), new String[]{"col1", "col3", "lastv"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.INT} + }, + new Object[]{ + "col1", List.of(0), new String[]{"col1", "lastv"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT} + } + }; + //@formatter:on + } + + /// The schema is case-insensitive by default and the Match row type names the partition columns as the query wrote + /// them rather than as they resolved, so the source order has to survive a case mismatch too. + @Test + public void testPartitionKeyOrderSurvivesACaseMismatch() { + MatchNode match = planMatch("SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY COL3, COL1 ORDER BY ts " + + "MEASURES LAST(X.col3) AS lastv PATTERN (X+) DEFINE X AS X.col3 > 0)"); + assertEquals(match.getPartitionKeys(), List.of(2, 0)); + } + + @Test + public void testMultiValuePartitionKeyIsRejectedDuringPlanning() { + RuntimeException e = expectThrows(RuntimeException.class, () -> planMatch( + "SELECT * FROM e MATCH_RECOGNIZE (PARTITION BY mcol1 ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (A) DEFINE A AS A.col3 > 0)")); + assertTrue(e.getMessage().contains("multi-value or ARRAY PARTITION BY columns: 'mcol1'"), e.getMessage()); + } + + @Test(dataProvider = "multiValueAggregateFunctions") + public void testMultiValueAggregateOperandIsRejectedDuringPlanning(String function) { + RuntimeException e = expectThrows(RuntimeException.class, () -> planMatch( + "SELECT * FROM e MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts MEASURES " + function + + "(A.mcol2) AS value PATTERN (A) DEFINE A AS A.col3 > 0)")); + assertTrue(e.getMessage().contains("applies aggregate '" + function + "' to a multi-value or ARRAY operand"), + e.getMessage()); + } + + @DataProvider(name = "multiValueAggregateFunctions") + public Object[][] multiValueAggregateFunctions() { + return new Object[][]{ + new Object[]{"SUM"}, new Object[]{"MIN"}, new Object[]{"MAX"}, new Object[]{"AVG"}, new Object[]{"COUNT"} + }; + } + + /// A pattern variable named after the row source alias makes an unqualified column reference bind to that variable + /// instead of to the SQL:2016 universal row pattern variable, silently changing both measure values and which rows + /// match. The information is gone by the time the RexNodes are built, so it has to be rejected on the SqlNode side. + @Test(dataProvider = "rowSourceAliasCollisionQueries") + public void testRowSourceAliasCollisionIsRejected(String sql) { + RuntimeException e = expectThrows(RuntimeException.class, () -> planMatch(sql)); + assertTrue(e.getMessage().contains("collides with the row source alias"), e.getMessage()); + } + + @DataProvider(name = "rowSourceAliasCollisionQueries") + public Object[][] rowSourceAliasCollisionQueries() { + return new Object[][]{ + new Object[]{ + "SELECT * FROM a AS A MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s PATTERN (A B+) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0)" + }, + // No explicit alias needed: the bare table name is the row source alias too. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s PATTERN (\"a\" B+) " + + "DEFINE \"a\" AS \"a\".col3 > 0, B AS B.col3 > 0)" + } + }; + } + + /// The negative control for the check above: the collision is exact-case, because the symbol table is a plain + /// `Map` lookup. A case-insensitive check would reject this query, which works correctly today. + @Test + public void testAliasDifferingOnlyInCaseStillBindsToTheUniversalSymbol() { + MatchNode match = planMatch("SELECT * FROM a AS aa MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s " + + "PATTERN (AA B+) DEFINE AA AS AA.col3 > 0, B AS B.col3 > 0)"); + List refs = collectPatternFieldRefs(match.getMeasures().get(0).getExpression()); + assertEquals(refs.size(), 1); + assertEquals(refs.get(0).getAlpha(), "aa"); + assertEquals(refs.get(0).getSymbolOrdinal(), RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL); + } + + @Test + public void testQuantifierBoundAboveTheCapIsRejected() { + int overTheCap = RelToPlanNodeConverter.MAX_PATTERN_QUANTIFIER_BOUND + 1; + RuntimeException e = expectThrows(RuntimeException.class, () -> planMatch( + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno PATTERN (A{1," + overTheCap + + "}) DEFINE A AS A.col3 > 0)")); + assertTrue(e.getMessage().contains("exceeds the maximum supported bound of " + + RelToPlanNodeConverter.MAX_PATTERN_QUANTIFIER_BOUND), e.getMessage()); + } + + @Test + public void testQuantifierBoundAtTheCapIsAccepted() { + int atTheCap = RelToPlanNodeConverter.MAX_PATTERN_QUANTIFIER_BOUND; + MatchNode match = planMatch( + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno PATTERN (A{1," + atTheCap + + "}) DEFINE A AS A.col3 > 0)"); + assertEquals(((RowPattern.Quantifier) match.getPattern()).getMaxRepeat(), atTheCap); + } + + /// Covers [PlanNodeToRelConverter#visitMatch]: the anchors move back into `strictStart` / `strictEnd`, the pattern + /// is rebuilt as Calcite's left deep `RexCall` tree, and `AFTER MATCH SKIP TO LAST` regains its target variable. + /// + /// The node is built by hand rather than planned from SQL because `RexExpressionUtils#toRexNode` cannot resolve + /// comparison operators (their function name is `GREATER_THAN`, which is not a FUNCTION-syntax operator); that is a + /// pre-existing limitation of this converter, shared with `FilterNode`, and unrelated to MATCH_RECOGNIZE. + @Test + public void testMatchNodeConvertsBackToLogicalMatch() { + DataSchema schema = new DataSchema(new String[]{"col1", "col3"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT}); + PlanNode input = new ExplainedNode(0, schema, null, List.of(), "Input", Map.of()); + // Calcite's Match requires at least one pattern definition, which the grammar guarantees (DEFINE is mandatory). + List symbols = + List.of(new PatternSymbol("A", new RexExpression.Literal(ColumnDataType.BOOLEAN, true)), + new PatternSymbol("B", null)); + RowPattern pattern = new RowPattern.Concat( + List.of(RowPattern.AnchorStart.INSTANCE, new RowPattern.Symbol(0), + new RowPattern.Quantifier(new RowPattern.Symbol(1), 2, 3, false), RowPattern.AnchorEnd.INSTANCE)); + List measures = + List.of(new MatchNode.Measure("ev", new RexExpression.PatternFieldRef(1, 1, "B"))); + MatchNode match = new MatchNode(0, schema, PlanNode.NodeHint.EMPTY, List.of(input), symbols, pattern, measures, + List.of(0), List.of(new RelFieldCollation(1)), MatchNode.AfterMatchSkipMode.TO_LAST, 1, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + + RelBuilder relBuilder = RelBuilder.create(Frameworks.newConfigBuilder() + .operatorTable(PinotOperatorTable.instance(true)) + .defaultSchema(CalciteSchema.createRootSchema(false).plus()) + .build()); + Match logicalMatch = findMatch(PlanNodeToRelConverter.convert(relBuilder, match)); + + assertNotNull(logicalMatch, "Expected a Match rel"); + assertTrue(logicalMatch.isStrictStart()); + assertTrue(logicalMatch.isStrictEnd()); + assertFalse(logicalMatch.isAllRows()); + // The anchors are gone from the pattern itself, and `{2,3}` is reluctant because 2 != 3. + assertEquals(logicalMatch.getPattern().toString(), "('A', PATTERN_QUANTIFIER('B', 2, 3, true))"); + assertEquals(logicalMatch.getAfter().toString(), "SKIP TO LAST('B')"); + // Only A has a DEFINE predicate; B matches every row and contributes no pattern definition. + assertEquals(logicalMatch.getPatternDefinitions().keySet(), Set.of("A")); + assertEquals(logicalMatch.getMeasures().keySet(), Set.of("ev")); + assertEquals(logicalMatch.getPartitionKeys().asList(), List.of(0)); + assertEquals(logicalMatch.getOrderKeys().getFieldCollations(), List.of(new RelFieldCollation(1))); + } + + private static List symbolNames(MatchNode match) { + return match.getPatternSymbols().stream().map(PatternSymbol::getName).collect(Collectors.toList()); + } + + private static List collectPatternFieldRefs(@Nullable RexExpression expression) { + List refs = new ArrayList<>(); + collectPatternFieldRefs(expression, refs); + return refs; + } + + private static void collectPatternFieldRefs(@Nullable RexExpression expression, + List refs) { + if (expression instanceof RexExpression.PatternFieldRef) { + refs.add((RexExpression.PatternFieldRef) expression); + } else if (expression instanceof RexExpression.FunctionCall) { + for (RexExpression operand : ((RexExpression.FunctionCall) expression).getFunctionOperands()) { + collectPatternFieldRefs(operand, refs); + } + } + } + + @Nullable + private static Match findMatch(RelNode rel) { + if (rel instanceof Match) { + return (Match) rel; + } + for (RelNode input : rel.getInputs()) { + Match match = findMatch(input); + if (match != null) { + return match; + } + } + return null; + } + + /// Plans {@code sql} through the full query environment and returns the single [MatchNode] in the resulting plan. + /// + /// These queries exercise pattern lowering rather than distribution, so most of them omit `PARTITION BY`, which + /// [org.apache.pinot.calcite.rel.rules.PinotMatchExchangeNodeInsertRule] rejects by default. The query option opts + /// into the resulting single-worker plan. + private MatchNode planMatch(String sql) { + SqlNodeAndOptions sqlNodeAndOptions = + new SqlNodeAndOptions(CalciteSqlParser.compileToSqlNodeAndOptions(sql).getSqlNode(), PinotSqlType.DQL, + QueryOptionsUtils.resolveCaseInsensitiveOptions( + Map.of(QueryOptionKey.ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY, "true"))); + DispatchableSubPlan subPlan = _queryEnvironment.compile(sql, sqlNodeAndOptions) + .planQuery(RANDOM_REQUEST_ID_GEN.nextLong()).getQueryPlan(); + List found = new ArrayList<>(); + for (DispatchablePlanFragment fragment : subPlan.getQueryStageMap().values()) { + fragment.getPlanFragment().getFragmentRoot().visit(new PlanNodeVisitor.DepthFirstVisitor() { + @Override + public Void visitMatch(MatchNode node, Void context) { + found.add(node); + return super.visitMatch(node, context); + } + + @Override + protected boolean traverseStageBoundary() { + return false; + } + }, null); + } + assertEquals(found.size(), 1, "Expected exactly one MatchNode in the plan of: " + sql); + return found.get(0); + } +} diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/plannode/MatchNodeTest.java b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/plannode/MatchNodeTest.java new file mode 100644 index 000000000000..a61aea3e8cb9 --- /dev/null +++ b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/plannode/MatchNodeTest.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.planner.plannode; + +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertSame; + + +public class MatchNodeTest { + private static final DataSchema DATA_SCHEMA = new DataSchema(new String[]{"sym", "startPrice"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.DOUBLE}); + private static final List SYMBOLS = + List.of(new PatternSymbol("A", null), new PatternSymbol("B", null), new PatternSymbol("C", null)); + + @Test + public void testExplain() { + assertEquals(buildNode(new RowPattern.Symbol(0)).explain(), "MATCH_RECOGNIZE"); + } + + @Test + public void testPatternStringRendersQuantifiers() { + // A* A+ A? A{3} A{3,} A{3,5}, and the reluctant form of each. + assertPattern("A*", + new RowPattern.Quantifier(new RowPattern.Symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true)); + assertPattern("A*?", + new RowPattern.Quantifier(new RowPattern.Symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, false)); + assertPattern("A+", + new RowPattern.Quantifier(new RowPattern.Symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true)); + assertPattern("A+?", + new RowPattern.Quantifier(new RowPattern.Symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, false)); + assertPattern("A?", new RowPattern.Quantifier(new RowPattern.Symbol(0), 0, 1, true)); + assertPattern("A??", new RowPattern.Quantifier(new RowPattern.Symbol(0), 0, 1, false)); + assertPattern("A{3}", new RowPattern.Quantifier(new RowPattern.Symbol(0), 3, 3, true)); + assertPattern("A{3,}", + new RowPattern.Quantifier(new RowPattern.Symbol(0), 3, RowPattern.Quantifier.UNBOUNDED, true)); + assertPattern("A{3,5}", new RowPattern.Quantifier(new RowPattern.Symbol(0), 3, 5, true)); + assertPattern("A{3,5}?", new RowPattern.Quantifier(new RowPattern.Symbol(0), 3, 5, false)); + } + + @Test + public void testPatternStringRendersNestedGrouping() { + // ^ (A B)* (B | C{2}) $ - a quantified concatenation must be parenthesized, an alternation parenthesizes itself. + RowPattern pattern = new RowPattern.Concat(List.of( + RowPattern.AnchorStart.INSTANCE, + new RowPattern.Quantifier(new RowPattern.Concat(List.of(new RowPattern.Symbol(0), new RowPattern.Symbol(1))), 0, + RowPattern.Quantifier.UNBOUNDED, true), + new RowPattern.Alternate(List.of(new RowPattern.Symbol(1), + new RowPattern.Quantifier(new RowPattern.Symbol(2), 2, 2, true))), + RowPattern.AnchorEnd.INSTANCE)); + assertPattern("^ (A B)* (B | C{2}) $", pattern); + } + + @Test + public void testWithInputs() { + MatchNode node = buildNode(new RowPattern.Symbol(0)); + PlanNode input = new TableScanNode(1, DATA_SCHEMA, PlanNode.NodeHint.EMPTY, new ArrayList<>(), "testTable", + List.of("sym", "price")); + + MatchNode withInput = (MatchNode) node.withInputs(List.of(input)); + assertEquals(withInput.getInputs(), List.of(input)); + assertSame(withInput.getPattern(), node.getPattern()); + assertEquals(withInput.getPatternSymbols(), node.getPatternSymbols()); + assertEquals(withInput.getMeasures(), node.getMeasures()); + assertEquals(withInput.getAfterMatchSkipMode(), node.getAfterMatchSkipMode()); + } + + @Test + public void testEqualsAndHashCode() { + MatchNode node1 = buildNode(new RowPattern.Symbol(0)); + MatchNode node2 = buildNode(new RowPattern.Symbol(0)); + assertEquals(node1, node2); + assertEquals(node1.hashCode(), node2.hashCode()); + + // A different pattern is a different node, even though everything else matches. + assertNotEquals(node1, buildNode(new RowPattern.Symbol(1))); + // A greedy and a reluctant quantifier select different matches and must never compare equal. + assertNotEquals(new RowPattern.Quantifier(new RowPattern.Symbol(0), 1, 2, true), + new RowPattern.Quantifier(new RowPattern.Symbol(0), 1, 2, false)); + // Alternation is ordered: the leftmost alternative wins, so a reordering is a different pattern. + assertNotEquals(new RowPattern.Alternate(List.of(new RowPattern.Symbol(0), new RowPattern.Symbol(1))), + new RowPattern.Alternate(List.of(new RowPattern.Symbol(1), new RowPattern.Symbol(0)))); + // Concatenation and alternation of the same children mean different things. + assertNotEquals(new RowPattern.Concat(List.of(new RowPattern.Symbol(0), new RowPattern.Symbol(1))), + new RowPattern.Alternate(List.of(new RowPattern.Symbol(0), new RowPattern.Symbol(1)))); + } + + /// A pattern field reference must never compare equal to the input ref with the same index: they read different + /// rows, and conflating them is exactly the degradation this class exists to prevent. + @Test + public void testPatternFieldRefIsNotAnInputRef() { + RexExpression.PatternFieldRef patternFieldRef = new RexExpression.PatternFieldRef(1, 0, "A"); + assertNotEquals(patternFieldRef, new RexExpression.InputRef(1)); + assertNotEquals(patternFieldRef, new RexExpression.PatternFieldRef(1, 1, "B")); + assertEquals(patternFieldRef.withSymbolOrdinal(1), new RexExpression.PatternFieldRef(1, 1, "A")); + } + + private static void assertPattern(String expected, RowPattern pattern) { + assertEquals(buildNode(pattern).getPatternString(), expected); + } + + private static MatchNode buildNode(RowPattern pattern) { + return new MatchNode(1, DATA_SCHEMA, PlanNode.NodeHint.EMPTY, new ArrayList<>(), SYMBOLS, pattern, + List.of(new MatchNode.Measure("startPrice", new RexExpression.PatternFieldRef(1, 0, "A"))), List.of(0), + List.of(new RelFieldCollation(1)), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + } +} diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/PlanNodeSerDeTest.java b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/PlanNodeSerDeTest.java index 0ba5b142eef3..8cab32ed3e8e 100644 --- a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/PlanNodeSerDeTest.java +++ b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/PlanNodeSerDeTest.java @@ -18,9 +18,14 @@ */ package org.apache.pinot.query.planner.serde; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.pinot.common.proto.Plan; import org.apache.pinot.common.utils.DataSchema; import org.apache.pinot.common.utils.DataSchema.ColumnDataType; import org.apache.pinot.query.QueryEnvironmentTestBase; @@ -31,11 +36,16 @@ import org.apache.pinot.query.planner.plannode.AggregateNode.AggType; import org.apache.pinot.query.planner.plannode.EnrichedJoinNode; import org.apache.pinot.query.planner.plannode.JoinNode; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; import org.apache.pinot.query.planner.plannode.PlanNode; +import org.apache.pinot.query.planner.plannode.RowPattern; import org.apache.pinot.query.planner.plannode.UnnestNode; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; public class PlanNodeSerDeTest extends QueryEnvironmentTestBase { @@ -131,4 +141,144 @@ public void testAggregateGroupingSetsSerDe() { assertEquals(deserialized.getGroupingSets(), groupingSets); assertEquals(deserialized, node); } + + /// Round-trips a MATCH_RECOGNIZE node whose pattern is `^ A (B{2,3} | C+?) D $`: nested alternation, a bounded + /// quantifier, a reluctant quantifier and both anchors. Also covers the pattern-variable symbol table, a DEFINE + /// predicate carrying a {@link RexExpression.PatternFieldRef}, MEASURES and `AFTER MATCH SKIP TO LAST C`. + @Test + public void testMatchNodeSerDe() { + MatchNode node = buildMatchNode(MatchNode.AfterMatchSkipMode.TO_LAST, 2); + + MatchNode deserialized = (MatchNode) PlanNodeDeserializer.process(PlanNodeSerializer.process(node)); + assertEquals(deserialized, node); + assertEquals(deserialized.getPatternString(), "^ A (B{2,3} | C+?) D $"); + assertEquals(deserialized.getAfterMatchSkipMode(), MatchNode.AfterMatchSkipMode.TO_LAST); + assertEquals(deserialized.getAfterMatchSkipToSymbolOrdinal(), 2); + assertEquals(deserialized.getRowsPerMatchMode(), MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + assertEquals(deserialized.getPartitionKeys(), List.of(0)); + assertEquals(deserialized.getCollations().size(), 1); + + // The pattern variable of a DEFINE reference must survive: degrading it to a plain InputRef would silently turn + // `B.price` into a read of the current row. + RexExpression definition = deserialized.getPatternSymbols().get(1).getDefinition(); + RexExpression.PatternFieldRef ref = + (RexExpression.PatternFieldRef) ((RexExpression.FunctionCall) definition).getFunctionOperands().get(0); + assertEquals(ref.getSymbolOrdinal(), 1); + assertEquals(ref.getIndex(), 1); + assertEquals(ref.getAlpha(), "B"); + + RexExpression.PatternFieldRef universalRef = + (RexExpression.PatternFieldRef) deserialized.getMeasures().get(1).getExpression(); + assertEquals(universalRef.getSymbolOrdinal(), RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL); + assertEquals(universalRef.getAlpha(), "prices"); + + // Variables without a DEFINE entry match every row and must round-trip as "no definition", not as a default. + assertNull(deserialized.getPatternSymbols().get(0).getDefinition()); + } + + @Test + public void testMatchNodeUsesPinnedWireFieldAndIsUnknownToAnOlderSchema() + throws Exception { + Plan.PlanNode serialized = PlanNodeSerializer.process( + buildMatchNode(MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL)); + + assertEquals(Plan.PlanNode.getDescriptor().findFieldByName("matchNode").getNumber(), 19); + Descriptors.Descriptor legacyPlanNodeDescriptor = buildLegacyPlanNodeDescriptor(); + DynamicMessage legacyPlanNode = DynamicMessage.parseFrom(legacyPlanNodeDescriptor, serialized.toByteArray()); + + // A server schema predating MATCH_RECOGNIZE preserves field 19 but does not select a node in its oneof. Its plan + // deserializer therefore observes NODE_NOT_SET and cannot execute the stage, which is why all servers must be + // upgraded before MATCH_RECOGNIZE queries are issued. + assertNull(legacyPlanNode.getOneofFieldDescriptor(legacyPlanNodeDescriptor.getOneofs().get(0))); + assertEquals(legacyPlanNode.getUnknownFields().getField(19).getLengthDelimitedList().size(), 1); + } + + private static Descriptors.Descriptor buildLegacyPlanNodeDescriptor() + throws Descriptors.DescriptorValidationException { + DescriptorProtos.DescriptorProto legacyUnnestNode = DescriptorProtos.DescriptorProto.newBuilder() + .setName("UnnestNode") + .build(); + DescriptorProtos.DescriptorProto legacyPlanNode = DescriptorProtos.DescriptorProto.newBuilder() + .setName("PlanNode") + .addOneofDecl(DescriptorProtos.OneofDescriptorProto.newBuilder().setName("node")) + .addField(DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName("unnestNode") + .setNumber(18) + .setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setType(DescriptorProtos.FieldDescriptorProto.Type.TYPE_MESSAGE) + .setTypeName(".legacy.UnnestNode") + .setOneofIndex(0)) + .build(); + DescriptorProtos.FileDescriptorProto legacyFile = DescriptorProtos.FileDescriptorProto.newBuilder() + .setName("legacy_plan.proto") + .setPackage("legacy") + .setSyntax("proto3") + .addMessageType(legacyUnnestNode) + .addMessageType(legacyPlanNode) + .build(); + return Descriptors.FileDescriptor.buildFrom(legacyFile, new Descriptors.FileDescriptor[0]) + .findMessageTypeByName("PlanNode"); + } + + /// `AFTER MATCH SKIP PAST LAST ROW` has no target variable. Ordinal 0 is a valid pattern variable, so "no target" + /// must not be encoded as the proto3 default of the field. + @Test + public void testMatchNodeWithoutSkipToSymbolSerDe() { + MatchNode node = buildMatchNode(MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL); + + MatchNode deserialized = (MatchNode) PlanNodeDeserializer.process(PlanNodeSerializer.process(node)); + assertEquals(deserialized, node); + assertEquals(deserialized.getAfterMatchSkipToSymbolOrdinal(), MatchNode.NO_SKIP_TO_SYMBOL); + } + + /// A pattern field reference that was never bound to a pattern symbol must not reach the wire: an ambiguous + /// reference would be resolved arbitrarily by the server and produce wrong-but-type-correct results. + @Test + public void testInvalidPatternFieldRefOrdinalsAreRejected() { + DataSchema dataSchema = new DataSchema(new String[]{"sym", "startPrice"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.DOUBLE}); + for (int invalidOrdinal : new int[]{RexExpression.PatternFieldRef.UNRESOLVED_SYMBOL_ORDINAL, -3, 1}) { + RexExpression.PatternFieldRef invalid = new RexExpression.PatternFieldRef(1, invalidOrdinal, "A"); + assertThrows(IllegalArgumentException.class, + () -> new MatchNode(1, dataSchema, PlanNode.NodeHint.EMPTY, new ArrayList<>(), + List.of(new PatternSymbol("A", null)), new RowPattern.Symbol(0), + List.of(new MatchNode.Measure("startPrice", invalid)), List.of(0), + List.of(new RelFieldCollation(1)), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL, MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH)); + } + } + + private static MatchNode buildMatchNode(MatchNode.AfterMatchSkipMode skipMode, int skipToSymbolOrdinal) { + // PATTERN (^ A (B{2,3} | C+?) D $) over the symbol table [A, B, C, D]. + List patternSymbols = List.of( + new PatternSymbol("A", null), + new PatternSymbol("B", new RexExpression.FunctionCall(ColumnDataType.BOOLEAN, "GREATER_THAN", + List.of(new RexExpression.PatternFieldRef(1, 1, "B"), + new RexExpression.Literal(ColumnDataType.DOUBLE, 100.0d)))), + new PatternSymbol("C", new RexExpression.FunctionCall(ColumnDataType.BOOLEAN, "LESS_THAN", + List.of(new RexExpression.PatternFieldRef(1, 2, "C"), + new RexExpression.Literal(ColumnDataType.DOUBLE, 100.0d)))), + new PatternSymbol("D", null)); + RowPattern pattern = new RowPattern.Concat(List.of( + RowPattern.AnchorStart.INSTANCE, + new RowPattern.Symbol(0), + new RowPattern.Alternate(List.of( + new RowPattern.Quantifier(new RowPattern.Symbol(1), 2, 3, true), + new RowPattern.Quantifier(new RowPattern.Symbol(2), 1, RowPattern.Quantifier.UNBOUNDED, false))), + new RowPattern.Symbol(3), + RowPattern.AnchorEnd.INSTANCE)); + List measures = List.of( + new MatchNode.Measure("startPrice", new RexExpression.PatternFieldRef(1, 0, "A")), + // For the universal ordinal, alpha is Calcite's row-source alias, not a pattern-variable name. + new MatchNode.Measure("lastUniversal", + new RexExpression.PatternFieldRef(1, RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL, "prices")), + new MatchNode.Measure("matchNum", + new RexExpression.FunctionCall(ColumnDataType.LONG, "MATCH_NUMBER", List.of()))); + DataSchema dataSchema = new DataSchema(new String[]{"sym", "startPrice", "lastUniversal", "matchNum"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.DOUBLE, ColumnDataType.DOUBLE, ColumnDataType.LONG}); + + return new MatchNode(1, dataSchema, PlanNode.NodeHint.EMPTY, new ArrayList<>(), patternSymbols, pattern, measures, + List.of(0), List.of(new RelFieldCollation(1)), skipMode, skipToSymbolOrdinal, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + } } diff --git a/pinot-query-planner/src/test/java/org/apache/pinot/query/validate/MatchRecognizeValidatorTest.java b/pinot-query-planner/src/test/java/org/apache/pinot/query/validate/MatchRecognizeValidatorTest.java new file mode 100644 index 000000000000..2dc56d71b311 --- /dev/null +++ b/pinot-query-planner/src/test/java/org/apache/pinot/query/validate/MatchRecognizeValidatorTest.java @@ -0,0 +1,352 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.validate; + +import javax.annotation.Nullable; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.pinot.query.QueryEnvironment; +import org.apache.pinot.query.QueryEnvironment.CompiledQuery; +import org.apache.pinot.query.QueryEnvironmentTestBase; +import org.apache.pinot.query.validate.MatchRecognizeValidator.UnsupportedMatchRecognizeException; +import org.apache.pinot.sql.parsers.CalciteSqlParser; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Tests for [MatchRecognizeValidator]: the SQL:2016 `AFTER MATCH` default fix, the rejection of constructs that are +/// not supported yet, and end to end planning of a v1 legal `MATCH_RECOGNIZE` query. +public class MatchRecognizeValidatorTest extends QueryEnvironmentTestBase { + + /// A fully featured v1 legal query. `%s` is where the `AFTER MATCH` clause goes (possibly empty). + private static final String QUERY_TEMPLATE = + "SELECT * FROM a MATCH_RECOGNIZE (" + + " PARTITION BY col1" + + " ORDER BY ts" + + " MEASURES MATCH_NUMBER() AS mno, CLASSIFIER() AS cls, FIRST(A.col3) AS startv, LAST(B.col3) AS endv" + + " ONE ROW PER MATCH" + + " %s" + + " PATTERN (A B+ C?)" + + " DEFINE A AS A.col3 > 0, B AS B.col3 > PREV(B.col3, 1), C AS C.col3 < NEXT(C.col3))"; + + @Test + public void testOmittedAfterMatchDefaultsToSkipPastLastRow() { + // Calcite's SqlToRelConverter substitutes SKIP TO NEXT ROW for an omitted AFTER MATCH clause, which produces + // overlapping matches. SQL:2016 (and Trino / Snowflake / Oracle) mandate SKIP PAST LAST ROW. + SqlMatchRecognize match = validate(String.format(QUERY_TEMPLATE, "")); + assertAfterOption(match, SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW); + } + + @Test + public void testExplicitSkipToNextRowIsPreserved() { + SqlMatchRecognize match = validate(String.format(QUERY_TEMPLATE, "AFTER MATCH SKIP TO NEXT ROW")); + assertAfterOption(match, SqlMatchRecognize.AfterOption.SKIP_TO_NEXT_ROW); + } + + @Test + public void testExplicitSkipPastLastRowIsPreserved() { + SqlMatchRecognize match = validate(String.format(QUERY_TEMPLATE, "AFTER MATCH SKIP PAST LAST ROW")); + assertAfterOption(match, SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW); + } + + @Test(dataProvider = "skipToVariableQueries") + public void testExplicitSkipToVariableIsPreserved(String afterClause, String expectedUnparsed) { + SqlMatchRecognize match = validate(String.format(QUERY_TEMPLATE, afterClause)); + SqlNode after = match.getAfter(); + assertNotNull(after); + assertEquals(after.toString().replace('\n', ' '), expectedUnparsed); + } + + @DataProvider(name = "skipToVariableQueries") + public Object[][] skipToVariableQueries() { + return new Object[][]{ + new Object[]{"AFTER MATCH SKIP TO FIRST B", "SKIP TO FIRST `B`"}, + new Object[]{"AFTER MATCH SKIP TO LAST B", "SKIP TO LAST `B`"} + }; + } + + @Test + public void testNestedMatchRecognizeIsRewritten() { + // The visitor must reach a MATCH_RECOGNIZE nested inside a sub-query, not just a top level FROM clause. + SqlMatchRecognize match = validate("SELECT * FROM (SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts " + + "MEASURES MATCH_NUMBER() AS mno PATTERN (A) DEFINE A AS A.col3 > 0))"); + assertAfterOption(match, SqlMatchRecognize.AfterOption.SKIP_PAST_LAST_ROW); + } + + @Test(dataProvider = "deferredConstructQueries") + public void testDeferredConstructIsRejected(String sql, String expectedMessageFragment) { + UnsupportedMatchRecognizeException e = expectThrows(UnsupportedMatchRecognizeException.class, () -> validate(sql)); + assertTrue(e.getMessage().contains(expectedMessageFragment), + "Expected message to contain '" + expectedMessageFragment + "' but was: " + e.getMessage()); + } + + @DataProvider(name = "deferredConstructQueries") + public Object[][] deferredConstructQueries() { + //@formatter:off + return new Object[][]{ + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts ALL ROWS PER MATCH PATTERN (A) DEFINE A AS A.col3 > 0)", + "ALL ROWS PER MATCH is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (A B) SUBSET S = (A, B) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0)", + "SUBSET is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (PERMUTE(A)) DEFINE A AS A.col3 > 0)", + "PERMUTE is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (A {- B -} C) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0, C AS C.col3 > 0)", + "Pattern exclusions '{- -}' are not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (A B) WITHIN INTERVAL '10' SECOND " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0)", + "WITHIN is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (A B+) " + + "DEFINE A AS A.col3 > 0, B AS SUM(B.col3) < 100)", + "Aggregate function 'SUM' is not supported yet in the MATCH_RECOGNIZE DEFINE clause" + }, + new Object[]{ + // Pinot specific aggregates are still unresolved at this point and are matched by name. + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts PATTERN (A B+) " + + "DEFINE A AS A.col3 > 0, B AS DISTINCTCOUNT(B.col3) < 100)", + "Aggregate function 'DISTINCTCOUNT' is not supported yet in the MATCH_RECOGNIZE DEFINE clause" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts NULLS FIRST PATTERN (A) DEFINE A AS A.col3 > 0)", + "NULLS FIRST / NULLS LAST is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts NULLS LAST PATTERN (A) DEFINE A AS A.col3 > 0)", + "NULLS FIRST / NULLS LAST is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts + 1 PATTERN (A) DEFINE A AS A.col3 > 0)", + "ORDER BY on expressions is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col3 + 1 ORDER BY ts PATTERN (A) " + + "DEFINE A AS A.col3 > 0)", + "PARTITION BY on expressions is not supported yet" + }, + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (PATTERN (A) DEFINE A AS A.col3 > 0)", + "MATCH_RECOGNIZE requires an ORDER BY clause" + }, + new Object[]{ + // Calcite does not confine the RUNNING / FINAL prefix operators to a MatchRecognizeScope, so registering + // them would otherwise let them leak into an ordinary projection. + "SELECT FINAL col3 FROM a", + "FINAL can only be used in the MEASURES or DEFINE clause" + }, + new Object[]{ + "SELECT RUNNING col3 FROM a", + "RUNNING can only be used in the MEASURES or DEFINE clause" + }, + new Object[]{ + // MEASURES is optional in SQL:2016, but Calcite then sets the MATCH_RECOGNIZE row type to the whole + // *input* row type, which is not the ONE ROW PER MATCH shape. The query used to plan and then die on the + // server with a message about an internal invariant. + "SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts PATTERN (A) DEFINE A AS A.col3 > 0)", + "MATCH_RECOGNIZE requires a MEASURES clause" + }, + new Object[]{ + // Calcite adds a measure to the row type only if its alias is not taken yet, so this one vanishes from + // the output altogether. + "SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts MEASURES LAST(A.col3) AS col1 " + + "PATTERN (A) DEFINE A AS A.col3 > 0)", + "measure alias 'col1' collides with a PARTITION BY column or an earlier measure alias" + }, + new Object[]{ + // Two measures with the same alias collapse through the same nameExists branch. + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(A.col3) AS v, FIRST(A.col3) AS v " + + "PATTERN (A) DEFINE A AS A.col3 > 0)", + "measure alias 'v' collides with a PARTITION BY column or an earlier measure alias" + }, + new Object[]{ + // The row source alias doubles as the alpha of an unqualified column reference, so a pattern variable of + // the same name silently steals every unqualified reference from the universal row pattern variable. + "SELECT * FROM a AS A MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s PATTERN (A B+) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0)", + "pattern variable 'A' collides with the row source alias 'A'" + }, + new Object[]{ + // Without an explicit alias the bare table name plays the same role. + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s PATTERN (\"a\" B+) " + + "DEFINE \"a\" AS \"a\".col3 > 0, B AS B.col3 > 0)", + "pattern variable 'a' collides with the row source alias 'a'" + } + }; + //@formatter:on + } + + @Test(dataProvider = "acceptedQueries") + public void testAcceptedQuery(String sql) { + assertNotNull(validate(sql)); + } + + @DataProvider(name = "acceptedQueries") + public Object[][] acceptedQueries() { + //@formatter:off + return new Object[][]{ + // Full pattern algebra: anchors, grouping, alternation, and all quantifier forms including reluctant ones. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno " + + "PATTERN (^ A{2,5}? (B | C)* D+ E? F{3} G{2,} $) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0, C AS C.col3 > 0, D AS D.col3 > 0, E AS E.col3 > 0, " + + "F AS F.col3 > 0, G AS G.col3 > 0)" + }, + // DESC is allowed on a plain column reference. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts DESC MEASURES MATCH_NUMBER() AS mno PATTERN (A) " + + "DEFINE A AS A.col3 > 0)" + }, + // A pattern variable used but not DEFINEd only warns; SQL:2016 says its condition defaults to TRUE. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno PATTERN (A B+ TYPO) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > 0)" + }, + // A pattern variable that happens to be named PERMUTE is legal as long as it is DEFINEd. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno PATTERN (PERMUTE A) " + + "DEFINE PERMUTE AS PERMUTE.col3 > 0, A AS A.col3 > 0)" + }, + // Cross-symbol references and PREV / NEXT with an explicit offset in DEFINE. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES MATCH_NUMBER() AS mno PATTERN (A B+) " + + "DEFINE A AS A.col3 > 0, B AS B.col3 > PREV(A.col3, 3) AND B.col3 < NEXT(A.col3, 2))" + }, + // A measure alias that differs from a PARTITION BY column only in case is a distinct column, because + // Calcite's RelDataTypeFactory.Builder#nameExists is case-sensitive. Rejecting it would break a query that + // works today. + new Object[]{ + "SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts MEASURES LAST(A.col3) AS COL1 " + + "PATTERN (A) DEFINE A AS A.col3 > 0)" + }, + // A pattern variable that differs from the row source alias only in case binds to the universal row pattern + // variable correctly, because the symbol table lookup is exact-case. + new Object[]{ + "SELECT * FROM a AS aa MATCH_RECOGNIZE (ORDER BY ts MEASURES LAST(col3) AS s PATTERN (AA B+) " + + "DEFINE AA AS AA.col3 > 0, B AS B.col3 > 0)" + } + }; + //@formatter:on + } + + @Test + public void testLegalQueryPlansToLogicalMatch() { + String explain = + _queryEnvironment.explainQuery("EXPLAIN PLAN FOR " + String.format(QUERY_TEMPLATE, ""), + RANDOM_REQUEST_ID_GEN.nextLong()); + assertTrue(explain.contains("LogicalMatch"), "Expected a LogicalMatch in the plan but got:\n" + explain); + } + + @Test + public void testDeferredConstructIsRejectedByQueryEnvironment() { + // The validator must be wired into the planning pipeline, so the error has to surface from compile() too. + RuntimeException e = expectThrows(RuntimeException.class, () -> _queryEnvironment.compile( + "SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts ALL ROWS PER MATCH PATTERN (A) DEFINE A AS A.col3 > 0)")); + assertTrue(e.getMessage().contains("ALL ROWS PER MATCH is not supported yet"), e.getMessage()); + } + + @Test(dataProvider = "physicalOptimizerOptions") + public void testPhysicalOptimizerIsRejectedWithAnActionableMessage(String options, String expectedMode) { + RuntimeException e = expectThrows(RuntimeException.class, + () -> _queryEnvironment.compile(options + String.format(QUERY_TEMPLATE, ""))); + assertTrue(e.getMessage().contains("MATCH_RECOGNIZE is not supported by the multi-stage physical optimizer" + + expectedMode), e.getMessage()); + assertTrue(e.getMessage().contains("usePhysicalOptimizer=false"), e.getMessage()); + } + + @DataProvider(name = "physicalOptimizerOptions") + public Object[][] physicalOptimizerOptions() { + return new Object[][]{ + new Object[]{"SET usePhysicalOptimizer=true; ", ""}, + new Object[]{"SET usePhysicalOptimizer=true; SET useLiteMode=true; ", " in lite mode"} + }; + } + + @Test + public void testPhysicalOptimizerQueryOptionCanOptBackIntoTheSupportedPlanner() { + try (CompiledQuery ignored = _queryEnvironment.compile( + "SET usePhysicalOptimizer=false; SET useLiteMode=true; " + String.format(QUERY_TEMPLATE, ""))) { + assertNotNull(ignored); + } + } + + @Test + public void testPhysicalOptimizerBrokerDefaultAndQueryOverride() { + QueryEnvironment physicalOptimizerByDefault = getQueryEnvironment(3, 1, 2, TABLE_SCHEMAS, + SERVER1_SEGMENTS, SERVER2_SEGMENTS, PARTITIONED_SEGMENTS_MAP, true); + RuntimeException e = expectThrows(RuntimeException.class, + () -> physicalOptimizerByDefault.compile(String.format(QUERY_TEMPLATE, ""))); + assertTrue(e.getMessage().contains("MATCH_RECOGNIZE is not supported by the multi-stage physical optimizer"), + e.getMessage()); + + try (CompiledQuery ignored = physicalOptimizerByDefault.compile( + "SET usePhysicalOptimizer=false; " + String.format(QUERY_TEMPLATE, ""))) { + assertNotNull(ignored); + } + } + + private static void assertAfterOption(SqlMatchRecognize match, SqlMatchRecognize.AfterOption expected) { + SqlNode after = match.getAfter(); + assertNotNull(after, "AFTER MATCH operand should have been populated by MatchRecognizeValidator"); + assertTrue(after instanceof SqlLiteral, "Expected a symbol literal but got: " + after); + assertEquals(((SqlLiteral) after).getValue(), expected); + } + + /// Parses {@code sql}, runs [MatchRecognizeValidator] over it, and returns the first `MATCH_RECOGNIZE` node found. + private static SqlMatchRecognize validate(String sql) { + SqlNode sqlNode = CalciteSqlParser.compileToSqlNodeAndOptions(sql).getSqlNode(); + sqlNode.accept(new MatchRecognizeValidator()); + SqlMatchRecognize match = findMatchRecognize(sqlNode); + assertNotNull(match, "Query does not contain a MATCH_RECOGNIZE clause: " + sql); + return match; + } + + @Nullable + private static SqlMatchRecognize findMatchRecognize(SqlNode root) { + SqlMatchRecognize[] found = new SqlMatchRecognize[1]; + root.accept(new SqlBasicVisitor() { + @Override + public Void visit(SqlCall call) { + if (call instanceof SqlMatchRecognize && found[0] == null) { + found[0] = (SqlMatchRecognize) call; + } + return super.visit(call); + } + }); + return found[0]; + } +} diff --git a/pinot-query-planner/src/test/resources/queries/MatchRecognizePlans.json b/pinot-query-planner/src/test/resources/queries/MatchRecognizePlans.json new file mode 100644 index 000000000000..a7c3027659ec --- /dev/null +++ b/pinot-query-planner/src/test/resources/queries/MatchRecognizePlans.json @@ -0,0 +1,68 @@ +{ + "match_recognize_exchange_planning_tests": { + "queries": [ + { + "description": "PARTITION BY single key: hash exchange on the partition key, collation prepends it before ORDER BY", + "sql": "EXPLAIN PLAN FOR SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts MEASURES FIRST(DOWN.ts) AS start_ts, LAST(UP.ts) AS end_ts ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW PATTERN (DOWN+ UP+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3), UP AS UP.col3 > PREV(UP.col3)) AS mr", + "output": [ + "Execution Plan", + "\nLogicalMatch(partition=[[0]], order=[[7]], outputFields=[[col1, start_ts, end_ts]], allRows=[false], after=[FLAG(SKIP PAST LAST ROW)], pattern=[(PATTERN_QUANTIFIER(_UTF-8'DOWN', 1, -1, false), PATTERN_QUANTIFIER(_UTF-8'UP', 1, -1, false))], isStrictStarts=[false], isStrictEnds=[false], subsets=[[]], patternDefinitions=[[<(PREV(DOWN.$2, 0), PREV(DOWN.$2, 1)), >(PREV(UP.$2, 0), PREV(UP.$2, 1))]], inputFields=[[col1, col2, col3, col4, col5, col6, col7, ts, ts_timestamp]])", + "\n PinotLogicalSortExchange(distribution=[hash[0]], collation=[[0, 7]], isSortOnSender=[false], isSortOnReceiver=[true])", + "\n PinotLogicalTableScan(table=[[default, a]])", + "\n" + ] + }, + { + "description": "PARTITION BY multiple keys: every partition key is prepended, ORDER BY keys keep their direction", + "sql": "EXPLAIN PLAN FOR SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1, col2 ORDER BY ts DESC, col3 MEASURES FIRST(DOWN.ts) AS start_ts PATTERN (DOWN+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3)) AS mr", + "output": [ + "Execution Plan", + "\nLogicalMatch(partition=[[0, 1]], order=[[7 DESC, 2]], outputFields=[[col1, col2, start_ts]], allRows=[false], after=[FLAG(SKIP PAST LAST ROW)], pattern=[PATTERN_QUANTIFIER(_UTF-8'DOWN', 1, -1, false)], isStrictStarts=[false], isStrictEnds=[false], subsets=[[]], patternDefinitions=[[<(PREV(DOWN.$2, 0), PREV(DOWN.$2, 1))]], inputFields=[[col1, col2, col3, col4, col5, col6, col7, ts, ts_timestamp]])", + "\n PinotLogicalSortExchange(distribution=[hash[0, 1]], collation=[[0, 1, 7 DESC, 2]], isSortOnSender=[false], isSortOnReceiver=[true])", + "\n PinotLogicalTableScan(table=[[default, a]])", + "\n" + ] + }, + { + "description": "ORDER BY key that is also a partition key is not repeated in the exchange collation", + "sql": "EXPLAIN PLAN FOR SELECT * FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY col1 DESC, ts MEASURES FIRST(DOWN.ts) AS start_ts PATTERN (DOWN+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3)) AS mr", + "output": [ + "Execution Plan", + "\nLogicalMatch(partition=[[0]], order=[[0 DESC, 7]], outputFields=[[col1, start_ts]], allRows=[false], after=[FLAG(SKIP PAST LAST ROW)], pattern=[PATTERN_QUANTIFIER(_UTF-8'DOWN', 1, -1, false)], isStrictStarts=[false], isStrictEnds=[false], subsets=[[]], patternDefinitions=[[<(PREV(DOWN.$2, 0), PREV(DOWN.$2, 1))]], inputFields=[[col1, col2, col3, col4, col5, col6, col7, ts, ts_timestamp]])", + "\n PinotLogicalSortExchange(distribution=[hash[0]], collation=[[0, 7]], isSortOnSender=[false], isSortOnReceiver=[true])", + "\n PinotLogicalTableScan(table=[[default, a]])", + "\n" + ] + }, + { + "description": "filter above MATCH_RECOGNIZE stays above it, the exchange stays directly below it", + "sql": "EXPLAIN PLAN FOR SELECT start_ts FROM a MATCH_RECOGNIZE (PARTITION BY col1 ORDER BY ts MEASURES FIRST(DOWN.ts) AS start_ts PATTERN (DOWN+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3)) AS mr WHERE start_ts > 0", + "output": [ + "Execution Plan", + "\nLogicalProject(start_ts=[$1])", + "\n LogicalFilter(condition=[>($1, 0)])", + "\n LogicalMatch(partition=[[0]], order=[[7]], outputFields=[[col1, start_ts]], allRows=[false], after=[FLAG(SKIP PAST LAST ROW)], pattern=[PATTERN_QUANTIFIER(_UTF-8'DOWN', 1, -1, false)], isStrictStarts=[false], isStrictEnds=[false], subsets=[[]], patternDefinitions=[[<(PREV(DOWN.$2, 0), PREV(DOWN.$2, 1))]], inputFields=[[col1, col2, col3, col4, col5, col6, col7, ts, ts_timestamp]])", + "\n PinotLogicalSortExchange(distribution=[hash[0]], collation=[[0, 7]], isSortOnSender=[false], isSortOnReceiver=[true])", + "\n PinotLogicalTableScan(table=[[default, a]])", + "\n" + ] + }, + { + "description": "no PARTITION BY is rejected by default because it funnels the whole table onto one worker", + "sql": "EXPLAIN PLAN FOR SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES FIRST(DOWN.ts) AS start_ts PATTERN (DOWN+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3)) AS mr", + "expectedException": "MATCH_RECOGNIZE without a PARTITION BY clause is not allowed" + }, + { + "description": "allowMatchRecognizeWithoutPartitionBy opts into the single-worker plan", + "sql": "SET allowMatchRecognizeWithoutPartitionBy='true'; EXPLAIN PLAN FOR SELECT * FROM a MATCH_RECOGNIZE (ORDER BY ts MEASURES FIRST(DOWN.ts) AS start_ts PATTERN (DOWN+) DEFINE DOWN AS DOWN.col3 < PREV(DOWN.col3)) AS mr", + "output": [ + "Execution Plan", + "\nLogicalMatch(partition=[[]], order=[[7]], outputFields=[[start_ts]], allRows=[false], after=[FLAG(SKIP PAST LAST ROW)], pattern=[PATTERN_QUANTIFIER(_UTF-8'DOWN', 1, -1, false)], isStrictStarts=[false], isStrictEnds=[false], subsets=[[]], patternDefinitions=[[<(PREV(DOWN.$2, 0), PREV(DOWN.$2, 1))]], inputFields=[[col1, col2, col3, col4, col5, col6, col7, ts, ts_timestamp]])", + "\n PinotLogicalSortExchange(distribution=[hash], collation=[[7]], isSortOnSender=[false], isSortOnReceiver=[true])", + "\n PinotLogicalTableScan(table=[[default, a]])", + "\n" + ] + } + ] + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java index 1e6f7e602429..ae109c190580 100644 --- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java @@ -38,6 +38,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -376,6 +377,11 @@ public ObjectNode visitUnnest(UnnestNode node, Context context) { return recursiveCase(node, MultiStageOperator.Type.UNNEST, context); } + @Override + public ObjectNode visitMatch(MatchNode node, Context context) { + return recursiveCase(node, MultiStageOperator.Type.MATCH, context); + } + public static class Context { private final int _parallelism; diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java index 88477e8b01c4..b1ae9bcc5993 100644 --- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/QueryRunner.java @@ -57,6 +57,7 @@ import org.apache.pinot.query.runtime.operator.LeafOperator; import org.apache.pinot.query.runtime.operator.MultiStageOperator; import org.apache.pinot.query.runtime.operator.OpChain; +import org.apache.pinot.query.runtime.operator.match.MatchLimits; import org.apache.pinot.query.runtime.plan.OpChainConverterDispatcher; import org.apache.pinot.query.runtime.plan.OpChainExecutionContext; import org.apache.pinot.query.runtime.plan.pipeline.PipelineBreakerExecutor; @@ -128,6 +129,10 @@ public class QueryRunner { @Nullable private WindowOverFlowMode _windowOverflowMode; @Nullable + private Integer _maxRowsInMatchPartition; + @Nullable + private Long _maxStepsPerMatchAttempt; + @Nullable private PhysicalTimeSeriesServerPlanVisitor _timeSeriesPhysicalPlanVisitor; /// Cluster-level decision on whether to send stats over the mailbox path, driven by the `SendStatsPredicate` /// at startup time. **May be overridden per-request** via the `KEY_OF_STATS_REPORTING_MODE` metadata key — @@ -188,6 +193,13 @@ public void init(PinotConfiguration serverConf, String instanceId, @Nullable Ins String windowOverflowModeStr = serverConf.getProperty(MultiStageQueryRunner.KEY_OF_WINDOW_OVERFLOW_MODE); _windowOverflowMode = windowOverflowModeStr != null ? WindowOverFlowMode.valueOf(windowOverflowModeStr) : null; + String maxRowsInMatchPartitionStr = serverConf.getProperty(MatchLimits.KEY_OF_MAX_ROWS_IN_MATCH_PARTITION); + _maxRowsInMatchPartition = + maxRowsInMatchPartitionStr != null ? Integer.parseInt(maxRowsInMatchPartitionStr) : null; + + String maxStepsPerMatchAttemptStr = serverConf.getProperty(MatchLimits.KEY_OF_MAX_STEPS_PER_MATCH_ATTEMPT); + _maxStepsPerMatchAttempt = maxStepsPerMatchAttemptStr != null ? Long.parseLong(maxStepsPerMatchAttemptStr) : null; + ExecutorService baseExecutorService = ExecutorServiceUtils.create(serverConf, Server.MULTISTAGE_EXECUTOR_CONFIG_PREFIX, "query-runner-on-" + port, Server.DEFAULT_MULTISTAGE_EXECUTOR_TYPE); @@ -546,9 +558,25 @@ private Map consolidateMetadata(Map customProper opChainMetadata.put(QueryOptionKey.WINDOW_OVERFLOW_MODE, windowOverflowMode.name()); } + applyMatchLimitDefaults(opChainMetadata, _maxRowsInMatchPartition, _maxStepsPerMatchAttempt); + return opChainMetadata; } + /// Applies MATCH_RECOGNIZE cluster defaults after query options have been canonicalized and consolidated. + /// Package-private for focused precedence coverage without initializing a complete query server. + static void applyMatchLimitDefaults(Map opChainMetadata, + @Nullable Integer maxRowsInMatchPartition, @Nullable Long maxStepsPerMatchAttempt) { + if (maxRowsInMatchPartition != null + && !opChainMetadata.containsKey(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION)) { + opChainMetadata.put(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, Integer.toString(maxRowsInMatchPartition)); + } + if (maxStepsPerMatchAttempt != null + && !opChainMetadata.containsKey(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT)) { + opChainMetadata.put(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, Long.toString(maxStepsPerMatchAttempt)); + } + } + public MailboxService getMailboxService() { return _mailboxService; } diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MatchOperator.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MatchOperator.java new file mode 100644 index 000000000000..63be9150ac10 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MatchOperator.java @@ -0,0 +1,431 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.common.datatable.StatMap; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.runtime.blocks.MseBlock; +import org.apache.pinot.query.runtime.blocks.RowHeapDataBlock; +import org.apache.pinot.query.runtime.operator.match.MatchExpression; +import org.apache.pinot.query.runtime.operator.match.MatchLimits; +import org.apache.pinot.query.runtime.operator.match.MatchTape; +import org.apache.pinot.query.runtime.operator.match.PartitionMatcher; +import org.apache.pinot.query.runtime.operator.match.PatternNfa; +import org.apache.pinot.query.runtime.operator.match.PatternToNfaCompiler; +import org.apache.pinot.query.runtime.operator.utils.TypeUtils; +import org.apache.pinot.query.runtime.plan.OpChainExecutionContext; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/// Evaluates SQL:2016 `MATCH_RECOGNIZE` (row pattern recognition) with `ONE ROW PER MATCH`. +/// +/// ## What it does per partition +/// +/// For every `PARTITION BY` partition, in `ORDER BY` order, it walks a scan position from the first row to +/// the last. At each position it asks [PartitionMatcher] for the preferred match starting exactly there. On a +/// match it emits one row - the partition key columns followed by the `MEASURES` - and then moves the scan +/// position according to the `AFTER MATCH SKIP` mode. On no match it moves one row forward. +/// +/// ## What it expects from the plan +/// +/// Like [WindowAggregateOperator], this operator does not sort. `PinotMatchExchangeNodeInsertRule` puts a +/// sort exchange underneath that hash distributes on the partition keys and sorts the receiver side on +/// `(partitionKeys..., orderKeys...)`, so rows arrive grouped by partition and ordered within a partition. The +/// operator therefore buffers one partition at a time and releases it at each boundary, and never reads +/// [MatchNode#getCollations()]: the ordering has already been established below it. +/// +/// The grouping half of that assumption is verified rather than trusted. Partition keys are compared directly on the +/// incoming rows and must be monotonically increasing, so a key that reappears after its partition was closed fails +/// without retaining every previously seen key. The ordering half is not re-checked per row, because an exchange that +/// grouped correctly but sorted incorrectly is not a failure mode the exchange can produce - losing the sort loses the +/// grouping too, which the partition-key order check already catches. +/// +/// ## Guardrails throw, they never truncate +/// +/// [QueryOptionKey#MAX_ROWS_IN_MATCH_PARTITION] bounds the rows buffered for a partition and +/// [QueryOptionKey#MAX_STEPS_PER_MATCH_ATTEMPT] bounds the backtracking of one match attempt. Both raise an error, +/// because a truncated pattern result is a wrong result that nothing in the response would flag. +/// +/// ## Not supported yet +/// +/// `ALL ROWS PER MATCH` is rejected here as well as during planning: this operator emits exactly one row per +/// match, so accepting it would silently return the wrong shape of result. +public class MatchOperator extends MultiStageOperator { + private static final Logger LOGGER = LoggerFactory.getLogger(MatchOperator.class); + private static final String EXPLAIN_NAME = "MATCH_RECOGNIZE"; + static final int MAX_OUTPUT_ROWS_PER_BLOCK = 1024; + + private final MultiStageOperator _input; + private final DataSchema _resultSchema; + private final ColumnDataType[] _resultStoredTypes; + private final int[] _partitionKeys; + private final int[] _partitionComparisonKeys; + private final List _patternSymbols; + private final MatchExpression[] _measures; + private final PartitionMatcher _matcher; + private final MatchNode.AfterMatchSkipMode _skipMode; + private final int _skipToSymbolOrdinal; + private final int _maxRowsInMatchPartition; + private final StatMap _statMap = new StatMap<>(StatKey.class); + + private List _partitionRows = new ArrayList<>(); + @Nullable + private List _inputRows; + private int _inputRowIndex; + private boolean _matchingPartition; + private int _scanStart; + private long _matchNumber; + @Nullable + private MseBlock.Eos _inputEos; + private int _numRows; + + public MatchOperator(OpChainExecutionContext context, MultiStageOperator input, DataSchema inputSchema, + MatchNode node) { + super(context); + if (node.getRowsPerMatchMode() != MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "ALL ROWS PER MATCH is not supported yet in MATCH_RECOGNIZE. Use ONE ROW PER MATCH (the default) and " + + "expose the per-match values you need through the MEASURES clause."); + } + _input = input; + _resultSchema = node.getDataSchema(); + _resultStoredTypes = _resultSchema.getStoredColumnDataTypes(); + List partitionKeys = node.getPartitionKeys(); + _partitionKeys = new int[partitionKeys.size()]; + for (int i = 0; i < _partitionKeys.length; i++) { + _partitionKeys[i] = partitionKeys.get(i); + ColumnDataType partitionKeyType = inputSchema.getColumnDataType(_partitionKeys[i]); + if (partitionKeyType.isArray()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY requires single-value columns, but column '" + + inputSchema.getColumnName(_partitionKeys[i]) + "' has type " + partitionKeyType); + } + } + // Calcite exposes the exchange distribution as an ImmutableBitSet, so the rule below this operator sorts + // partition fields by input ordinal. Preserve SQL declaration order in _partitionKeys for the output schema, but + // validate monotonic input against the order the exchange actually produces. + _partitionComparisonKeys = _partitionKeys.clone(); + Arrays.sort(_partitionComparisonKeys); + _patternSymbols = node.getPatternSymbols(); + List measures = node.getMeasures(); + _measures = new MatchExpression[measures.size()]; + for (int i = 0; i < _measures.length; i++) { + _measures[i] = MatchExpression.compile(measures.get(i).getExpression(), inputSchema); + } + if (_resultSchema.size() != _partitionKeys.length + _measures.length) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE output schema " + _resultSchema + " does not match " + _partitionKeys.length + + " partition key(s) plus " + _measures.length + " measure(s)"); + } + _skipMode = node.getAfterMatchSkipMode(); + _skipToSymbolOrdinal = node.getAfterMatchSkipToSymbolOrdinal(); + + PatternNfa nfa = PatternToNfaCompiler.compile(node.getPattern()); + _maxRowsInMatchPartition = + MatchLimits.getMaxRowsInMatchPartition(context.getOpChainMetadata(), node.getNodeHint()); + long maxStepsPerMatchAttempt = + MatchLimits.getMaxStepsPerMatchAttempt(context.getOpChainMetadata(), node.getNodeHint()); + _matcher = new PartitionMatcher(nfa, _patternSymbols, inputSchema, maxStepsPerMatchAttempt, + this::checkTerminationAndSampleUsage); + } + + @Override + protected Logger logger() { + return LOGGER; + } + + @Override + public List getChildOperators() { + return List.of(_input); + } + + @Override + public Type getOperatorType() { + return Type.MATCH; + } + + @Override + public String toExplainString() { + return EXPLAIN_NAME; + } + + @Override + public void registerExecution(long time, int numRows, long memoryUsedBytes, long gcTimeMs) { + _statMap.merge(StatKey.EXECUTION_TIME_MS, time); + _statMap.merge(StatKey.EMITTED_ROWS, numRows); + _statMap.merge(StatKey.ALLOCATED_MEMORY_BYTES, memoryUsedBytes); + _statMap.merge(StatKey.GC_TIME_MS, gcTimeMs); + } + + @Override + public StatMap copyStatMaps() { + return new StatMap<>(_statMap); + } + + @Override + protected MseBlock getNextBlock() { + List outputRows = new ArrayList<>(MAX_OUTPUT_ROWS_PER_BLOCK); + while (outputRows.size() < MAX_OUTPUT_ROWS_PER_BLOCK) { + if (_matchingPartition) { + emitMatches(outputRows); + continue; + } + + if (_inputRows != null) { + consumeInputRowsUntilPartitionBoundary(); + continue; + } + + if (_inputEos != null) { + if (outputRows.isEmpty()) { + return _inputEos; + } + break; + } + + MseBlock block = _input.nextBlock(); + if (block.isData()) { + _inputRows = ((MseBlock.Data) block).asRowHeap().getRows(); + _inputRowIndex = 0; + continue; + } + + _inputEos = (MseBlock.Eos) block; + if (_inputEos.isError()) { + return _inputEos; + } + if (!_partitionRows.isEmpty()) { + startMatchingPartition(); + } + } + return new RowHeapDataBlock(outputRows, _resultSchema); + } + + /// Consumes rows without allocating extracted keys. A boundary row stays in the input block until the preceding + /// partition has been fully matched and emitted. + private void consumeInputRowsUntilPartitionBoundary() { + assert _inputRows != null; + while (_inputRowIndex < _inputRows.size()) { + Object[] row = _inputRows.get(_inputRowIndex); + if (!_partitionRows.isEmpty()) { + int comparison = comparePartitionKeys(_partitionRows.get(0), row); + if (comparison > 0) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE input is not grouped by partition key in ascending order: partition " + + partitionKeyToString(row) + " appeared after " + partitionKeyToString(_partitionRows.get(0))); + } + if (comparison < 0) { + startMatchingPartition(); + return; + } + } + if (_partitionRows.size() >= _maxRowsInMatchPartition) { + throw QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.asException( + "MATCH_RECOGNIZE partition exceeds the maximum of " + _maxRowsInMatchPartition + + " rows. Partition on a column with more distinct values, or raise the '" + + QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION + "' query option."); + } + _partitionRows.add(row); + _inputRowIndex++; + _numRows++; + checkTerminationAndSampleUsagePeriodically(_numRows, EXPLAIN_NAME); + } + _inputRows = null; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private int comparePartitionKeys(Object[] leftRow, Object[] rightRow) { + for (int partitionKey : _partitionComparisonKeys) { + Object left = leftRow[partitionKey]; + Object right = rightRow[partitionKey]; + if (left == null) { + if (right == null) { + continue; + } + return 1; + } + if (right == null) { + return -1; + } + if (!(left instanceof Comparable)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY value is not sortable: " + left.getClass().getName()); + } + final int comparison; + try { + comparison = ((Comparable) left).compareTo(right); + } catch (ClassCastException e) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE PARTITION BY values have incompatible types: " + left.getClass().getName() + " and " + + right.getClass().getName()); + } + if (comparison != 0) { + return comparison; + } + } + return 0; + } + + private String partitionKeyToString(Object[] row) { + StringBuilder builder = new StringBuilder("["); + for (int i = 0; i < _partitionKeys.length; i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(row[_partitionKeys[i]]); + } + return builder.append(']').toString(); + } + + private void startMatchingPartition() { + _matchingPartition = true; + _scanStart = 0; + _matchNumber = 0; + } + + /// Resumes scanning the buffered partition until it is exhausted or the current output block is full. + private void emitMatches(List outputRows) { + List rows = _partitionRows; + int numPartitionRows = rows.size(); + MatchTape tape = _matcher.getTape(); + while (_scanStart < numPartitionRows && outputRows.size() < MAX_OUTPUT_ROWS_PER_BLOCK) { + checkTerminationAndSampleUsage(); + int endPos = _matcher.match(rows, _scanStart, _matchNumber + 1); + if (endPos == PartitionMatcher.NO_MATCH) { + _scanStart++; + continue; + } + _matchNumber++; + outputRows.add(buildOutputRow(rows, tape)); + _scanStart = nextScanStart(_scanStart, endPos, tape); + } + if (_scanStart >= numPartitionRows) { + _matcher.releasePartition(); + _partitionRows = new ArrayList<>(); + _matchingPartition = false; + } + } + + /// Builds the output row of one match: the partition key columns, which are constant across the partition, followed + /// by the measures evaluated against the completed match. + private Object[] buildOutputRow(List rows, MatchTape tape) { + Object[] outputRow = new Object[_partitionKeys.length + _measures.length]; + Object[] anyPartitionRow = rows.get(0); + for (int i = 0; i < _partitionKeys.length; i++) { + outputRow[i] = anyPartitionRow[_partitionKeys[i]]; + } + for (int i = 0; i < _measures.length; i++) { + outputRow[_partitionKeys.length + i] = _measures[i].evaluate(tape); + } + TypeUtils.convertRow(outputRow, _resultStoredTypes); + return outputRow; + } + + /// Where pattern matching resumes after a match that covered `[scanStart, endPos)`. + /// + /// An empty match always resumes at the next row: every skip mode would otherwise resume exactly where it + /// started and loop forever. `SKIP TO FIRST` / `SKIP TO LAST` that would not make progress is an error + /// in SQL:2016 rather than a silently adjusted position, so it is reported as one. + private int nextScanStart(int scanStart, int endPos, MatchTape tape) { + if (endPos == scanStart) { + if (_skipMode == MatchNode.AfterMatchSkipMode.TO_FIRST + || _skipMode == MatchNode.AfterMatchSkipMode.TO_LAST) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "AFTER MATCH SKIP TO " + skipTargetName() + " cannot be applied to the empty match at row " + scanStart + + " of its partition, because no row is mapped to that pattern variable. Make the PATTERN require " + + "at least one row, or use AFTER MATCH SKIP PAST LAST ROW."); + } + return scanStart + 1; + } + switch (_skipMode) { + case PAST_LAST_ROW: + return endPos; + case TO_NEXT_ROW: + return scanStart + 1; + case TO_FIRST: + return skipToRow(tape.firstRow(_skipToSymbolOrdinal, 0), scanStart, "FIRST"); + case TO_LAST: + return skipToRow(tape.lastRow(_skipToSymbolOrdinal, 0), scanStart, "LAST"); + default: + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported MATCH_RECOGNIZE AFTER MATCH SKIP mode: " + _skipMode); + } + } + + private int skipToRow(int targetRow, int scanStart, String position) { + if (targetRow == MatchTape.NO_ROW) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "AFTER MATCH SKIP TO " + position + " " + skipTargetName() + " failed: no row of the match starting at row " + + scanStart + " of its partition is mapped to that pattern variable."); + } + if (targetRow <= scanStart) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "AFTER MATCH SKIP TO " + position + " " + skipTargetName() + + " would resume at the first row of the match it just skipped, which would never terminate. Skip to a " + + "pattern variable that cannot be mapped to the first row of the match, or use AFTER MATCH SKIP PAST " + + "LAST ROW."); + } + return targetRow; + } + + private String skipTargetName() { + return _skipToSymbolOrdinal >= 0 && _skipToSymbolOrdinal < _patternSymbols.size() + ? _patternSymbols.get(_skipToSymbolOrdinal).getName() : ""; + } + + public enum StatKey implements StatMap.Key { + EXECUTION_TIME_MS(StatMap.Type.LONG) { + @Override + public boolean includeDefaultInJson() { + return true; + } + }, + EMITTED_ROWS(StatMap.Type.LONG) { + @Override + public boolean includeDefaultInJson() { + return true; + } + }, + /// Allocated memory in bytes for this operator or its children in the same stage. + ALLOCATED_MEMORY_BYTES(StatMap.Type.LONG), + /// Time spent on GC while this operator or its children in the same stage were running. + GC_TIME_MS(StatMap.Type.LONG); + + private final StatMap.Type _type; + + StatKey(StatMap.Type type) { + _type = type; + } + + @Override + public StatMap.Type getType() { + return _type; + } + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java index 0bead828f0fc..73388a296404 100644 --- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java @@ -504,10 +504,18 @@ public void mergeInto(BrokerResponseNativeV2 response, StatMap map) { StatMap stats = (StatMap) map; response.mergeMaxRowsInOperator(stats.getLong(RepeatOperator.StatKey.EMITTED_ROWS)); } + }, + MATCH(17, MatchOperator.StatKey.class) { + @Override + public void mergeInto(BrokerResponseNativeV2 response, StatMap map) { + @SuppressWarnings("unchecked") + StatMap stats = (StatMap) map; + response.mergeMaxRowsInOperator(stats.getLong(MatchOperator.StatKey.EMITTED_ROWS)); + } }; // When adding new operator types, update MAX_ID if the new ID exceeds the current max - private static final int MAX_ID = 16; + private static final int MAX_ID = 17; private static final Type[] ID_TO_TYPE = new Type[MAX_ID + 1]; static { diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchExpression.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchExpression.java new file mode 100644 index 000000000000..badba305f1f3 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchExpression.java @@ -0,0 +1,415 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import it.unimi.dsi.fastutil.ints.IntSet; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.runtime.operator.operands.TransformOperand; +import org.apache.pinot.query.runtime.operator.operands.TransformOperandFactory; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.spi.exception.QueryErrorCode; + + +/// A compiled MEASURES item or DEFINE predicate, evaluated against the state of one match. +/// +/// ## How it reuses Pinot's row expression evaluation +/// +/// A MATCH_RECOGNIZE expression such as `PREV(A.price, 1) > B.price * 2` is only *partly* match specific: +/// the navigations `PREV(A.price, 1)` and `B.price` depend on the match, but the `>` and the +/// `*` are ordinary scalar operators. Compilation therefore splits the expression in two: +/// 1. every maximal match specific sub-expression - a navigation, `CLASSIFIER()`, `MATCH_NUMBER()` or +/// a single variable aggregate - becomes a [MatchTerm] bound to a slot of a synthetic row; +/// 2. what is left is a plain [RexExpression] over those slots, compiled by +/// [TransformOperandFactory] exactly like any other Pinot row expression. +/// +/// The example compiles to slots `[PREV(A.price,1), B.price]` plus the operand `$0 > $1 * 2`. Comparisons, +/// boolean connectives, arithmetic, casts, null semantics and every scalar UDF therefore behave exactly as they do +/// elsewhere in the multi-stage engine, and this class does not reimplement any of them. +/// +/// `RUNNING` and `FINAL` are stripped during compilation: with ONE ROW PER MATCH the measures are +/// computed once the match is complete, and running semantics evaluated at the last row of a match coincide with final +/// semantics. Both modifiers therefore have no effect and are unwrapped rather than rejected. +/// +/// Not thread safe: the slot array is reused between evaluations, so one instance belongs to one operator. +public class MatchExpression { + /// Navigation and match state functions, as Calcite names them in the converted expression. + private static final String PREV = "PREV"; + private static final String NEXT = "NEXT"; + private static final String FIRST = "FIRST"; + private static final String LAST = "LAST"; + private static final String CLASSIFIER = "CLASSIFIER"; + private static final String MATCH_NUMBER = "MATCH_NUMBER"; + private static final String RUNNING = "RUNNING"; + private static final String FINAL = "FINAL"; + private static final Set NAVIGATION_FUNCTIONS = Set.of(PREV, NEXT, FIRST, LAST); + private static final Set MATCH_FUNCTIONS = + Set.of(PREV, NEXT, FIRST, LAST, CLASSIFIER, MATCH_NUMBER, RUNNING, FINAL); + + private final TransformOperand _operand; + private final List _terms; + private final Object[] _slots; + + private MatchExpression(TransformOperand operand, List terms) { + _operand = operand; + _terms = terms; + _slots = new Object[terms.size()]; + } + + /// Compiles `expression`, which addresses columns of `inputSchema` through pattern field references. + /// + /// @throws org.apache.pinot.spi.exception.QueryException if the expression uses a construct that is not supported + /// yet. Nothing is dropped or approximated, because either would return wrong rows instead of an error. + public static MatchExpression compile(RexExpression expression, DataSchema inputSchema) { + List terms = new ArrayList<>(); + RexExpression slotExpression = rewrite(expression, inputSchema, terms); + int numTerms = terms.size(); + String[] slotNames = new String[numTerms]; + ColumnDataType[] slotTypes = new ColumnDataType[numTerms]; + for (int i = 0; i < numTerms; i++) { + slotNames[i] = "$" + i; + slotTypes[i] = terms.get(i).getResultType(); + } + TransformOperand operand = + TransformOperandFactory.getTransformOperand(slotExpression, new DataSchema(slotNames, slotTypes)); + return new MatchExpression(operand, terms); + } + + /// Evaluates this expression against the current state of `tape`. + @Nullable + public Object evaluate(MatchTape tape) { + for (int i = 0; i < _terms.size(); i++) { + _slots[i] = _terms.get(i).evaluate(tape); + } + return _operand.apply(_slots); + } + + /// Evaluates this expression as a DEFINE predicate. SQL three valued logic collapses to false here: a row is mapped + /// to a pattern variable only if its condition is definitely true. + public boolean test(MatchTape tape) { + Object value = evaluate(tape); + if (value == null) { + return false; + } + if (value instanceof Boolean) { + return (Boolean) value; + } + return ((Number) value).intValue() != 0; + } + + /// Replaces every match specific sub-expression of `expression` with an input reference to a freshly appended + /// slot of `terms`, and returns the resulting expression over the synthetic slot row. + private static RexExpression rewrite(RexExpression expression, DataSchema inputSchema, List terms) { + if (expression instanceof RexExpression.PatternFieldRef) { + // A bare column reference such as `A.price` is `LAST(A.price, 0)` per SQL:2016. + RexExpression.PatternFieldRef ref = (RexExpression.PatternFieldRef) expression; + ColumnDataType sourceType = columnType(inputSchema, ref); + return addTerm(terms, new MatchTerm.Navigation(ref.getSymbolOrdinal(), true, 0, 0, ref.getIndex(), + sourceType, sourceType)); + } + if (expression instanceof RexExpression.InputRef) { + // Column references inside MEASURES / DEFINE always arrive as pattern field references. A plain input + // reference would index the synthetic slot row instead of the input row, so refuse it rather than read the + // wrong column. + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported column reference in a MATCH_RECOGNIZE MEASURES or DEFINE expression: " + expression + + ". Qualify the column with a pattern variable, e.g. 'A.column'."); + } + if (!(expression instanceof RexExpression.FunctionCall)) { + return expression; + } + + RexExpression.FunctionCall call = (RexExpression.FunctionCall) expression; + String functionName = call.getFunctionName(); + if (RUNNING.equals(functionName) || FINAL.equals(functionName)) { + return rewrite(singleOperand(call), inputSchema, terms); + } + if (CLASSIFIER.equals(functionName)) { + if (!call.getFunctionOperands().isEmpty()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "CLASSIFIER with a pattern variable argument is not supported yet in MATCH_RECOGNIZE. Use the no " + + "argument form CLASSIFIER()."); + } + return addTerm(terms, MatchTerm.Classifier.INSTANCE); + } + if (MATCH_NUMBER.equals(functionName)) { + return addTerm(terms, MatchTerm.MatchNumber.INSTANCE); + } + if (NAVIGATION_FUNCTIONS.contains(functionName)) { + // Calcite distributes navigation over an expression, so `LAST(A.price * 2)` arrives as + // `LAST(A.price, 0) * LAST(2, 0)`. Navigating a constant is the constant itself. + if (!containsPatternFieldRef(call)) { + return rewrite(singleOperand(call), inputSchema, terms); + } + return addTerm(terms, navigation(call, inputSchema)); + } + if (AggregationFunctionType.isAggregationFunction(functionName)) { + return addTerm(terms, aggregate(call, inputSchema)); + } + + List operands = call.getFunctionOperands(); + List rewritten = new ArrayList<>(operands.size()); + for (RexExpression operand : operands) { + rewritten.add(rewrite(operand, inputSchema, terms)); + } + return new RexExpression.FunctionCall(call.getDataType(), functionName, rewritten, call.isDistinct(), + call.isIgnoreNulls()); + } + + private static RexExpression addTerm(List terms, MatchTerm term) { + terms.add(term); + return new RexExpression.InputRef(terms.size() - 1); + } + + /// Flattens a nest of navigation calls such as `PREV(LAST(A.price, 1), 2)` into a single + /// [MatchTerm.Navigation]: at most one logical step (`FIRST` / `LAST`) selects a row of the match, + /// and the `PREV` / `NEXT` offsets around it add up into one physical delta. + private static MatchTerm.Navigation navigation(RexExpression.FunctionCall call, DataSchema inputSchema) { + boolean fromEnd = true; + int logicalOffset = 0; + boolean logicalSeen = false; + long physicalDelta = 0; + RexExpression current = call; + while (current instanceof RexExpression.FunctionCall) { + RexExpression.FunctionCall currentCall = (RexExpression.FunctionCall) current; + String functionName = currentCall.getFunctionName(); + if (RUNNING.equals(functionName) || FINAL.equals(functionName)) { + current = singleOperand(currentCall); + continue; + } + if (!NAVIGATION_FUNCTIONS.contains(functionName)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported expression inside a MATCH_RECOGNIZE row pattern navigation: '" + currentCall + + "'. Only a column reference qualified by a pattern variable, optionally wrapped in " + + "FIRST / LAST / PREV / NEXT, is supported."); + } + int offset = navigationOffset(currentCall); + try { + if (PREV.equals(functionName)) { + physicalDelta = Math.subtractExact(physicalDelta, offset); + } else if (NEXT.equals(functionName)) { + physicalDelta = Math.addExact(physicalDelta, offset); + } else { + if (logicalSeen) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Nested FIRST / LAST is not supported in MATCH_RECOGNIZE: '" + call + "'."); + } + logicalSeen = true; + fromEnd = LAST.equals(functionName); + logicalOffset = offset; + } + } catch (ArithmeticException e) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The combined PREV / NEXT offset in a MATCH_RECOGNIZE row pattern navigation exceeds the supported " + + "range of " + Integer.MIN_VALUE + " to " + Integer.MAX_VALUE + ": '" + call + "'.", e); + } + current = currentCall.getFunctionOperands().get(0); + } + if (!(current instanceof RexExpression.PatternFieldRef)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported operand of a MATCH_RECOGNIZE row pattern navigation: '" + call + + "'. Expecting a column reference qualified by a pattern variable."); + } + RexExpression.PatternFieldRef ref = (RexExpression.PatternFieldRef) current; + ColumnDataType sourceType = columnType(inputSchema, ref); + try { + return new MatchTerm.Navigation(ref.getSymbolOrdinal(), fromEnd, logicalOffset, Math.toIntExact(physicalDelta), + ref.getIndex(), sourceType, call.getDataType()); + } catch (ArithmeticException e) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The combined PREV / NEXT offset in a MATCH_RECOGNIZE row pattern navigation exceeds the supported " + + "range of " + Integer.MIN_VALUE + " to " + Integer.MAX_VALUE + ": '" + call + "'.", e); + } + } + + /// The offset operand of a navigation call. `PREV` and `NEXT` default to one row, `FIRST` and + /// `LAST` to the first / last row itself. + private static int navigationOffset(RexExpression.FunctionCall call) { + List operands = call.getFunctionOperands(); + if (operands.size() < 2) { + String functionName = call.getFunctionName(); + return PREV.equals(functionName) || NEXT.equals(functionName) ? 1 : 0; + } + RexExpression offset = operands.get(1); + if (!(offset instanceof RexExpression.Literal)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The offset of a MATCH_RECOGNIZE row pattern navigation must be a constant, got: '" + offset + "'."); + } + Object value = ((RexExpression.Literal) offset).getValue(); + if (!(value instanceof Number)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The offset of a MATCH_RECOGNIZE row pattern navigation must be an integer, got: '" + value + "'."); + } + BigDecimal decimalValue; + if (value instanceof BigDecimal) { + decimalValue = (BigDecimal) value; + } else if (value instanceof BigInteger) { + decimalValue = new BigDecimal((BigInteger) value); + } else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { + decimalValue = BigDecimal.valueOf(((Number) value).longValue()); + } else { + double doubleValue = ((Number) value).doubleValue(); + if (!Double.isFinite(doubleValue)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The offset of a MATCH_RECOGNIZE row pattern navigation must be a finite integer, got: '" + value + "'."); + } + decimalValue = BigDecimal.valueOf(doubleValue); + } + if (decimalValue.signum() < 0) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The offset of a MATCH_RECOGNIZE row pattern navigation must not be negative, got: " + value + "."); + } + try { + return decimalValue.intValueExact(); + } catch (ArithmeticException e) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "The offset of a MATCH_RECOGNIZE row pattern navigation must be an exact integer between 0 and " + + Integer.MAX_VALUE + ", got: '" + value + "'.", e); + } + } + + /// Compiles a single variable aggregate. The aggregated expression is compiled against the input schema so that it + /// can be evaluated per row of the match, which is why `SUM(A.price * A.quantity)` works. + private static MatchTerm.Aggregate aggregate(RexExpression.FunctionCall call, DataSchema inputSchema) { + String functionName = call.getFunctionName(); + if (call.isDistinct()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "DISTINCT is not supported in a MATCH_RECOGNIZE MEASURES aggregate: '" + call + "'."); + } + MatchTerm.Aggregate.Kind kind = MatchTerm.Aggregate.Kind.resolveOrThrow(functionName); + List operands = call.getFunctionOperands(); + if (operands.isEmpty()) { + if (kind != MatchTerm.Aggregate.Kind.COUNT) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Aggregate '" + functionName + "' requires an argument in a MATCH_RECOGNIZE MEASURES clause."); + } + // COUNT(*) counts every row of the match, whatever variable it is mapped to. + return new MatchTerm.Aggregate(kind, RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL, null, + call.getDataType()); + } + if (operands.size() != 1) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Multi argument aggregate '" + functionName + "' is not supported in a MATCH_RECOGNIZE MEASURES clause."); + } + RexExpression argument = operands.get(0); + if (containsMatchFunction(argument)) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Row pattern navigation inside a MATCH_RECOGNIZE MEASURES aggregate is not supported yet: '" + call + "'."); + } + IntSet symbolOrdinals = new IntOpenHashSet(); + collectSymbolOrdinals(argument, symbolOrdinals); + if (symbolOrdinals.size() > 1) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Only single variable aggregates are supported in a MATCH_RECOGNIZE MEASURES clause, but '" + call + + "' references " + symbolOrdinals.size() + " pattern variables."); + } + int symbolOrdinal = symbolOrdinals.isEmpty() ? RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL + : symbolOrdinals.iterator().nextInt(); + TransformOperand argumentOperand = + TransformOperandFactory.getTransformOperand(toRowExpression(argument), inputSchema); + return new MatchTerm.Aggregate(kind, symbolOrdinal, argumentOperand, call.getDataType()); + } + + /// Rewrites pattern field references into plain input references so the expression can be evaluated against a single + /// input row. + private static RexExpression toRowExpression(RexExpression expression) { + if (expression instanceof RexExpression.PatternFieldRef) { + return new RexExpression.InputRef(((RexExpression.PatternFieldRef) expression).getIndex()); + } + if (!(expression instanceof RexExpression.FunctionCall)) { + return expression; + } + RexExpression.FunctionCall call = (RexExpression.FunctionCall) expression; + List operands = call.getFunctionOperands(); + List rewritten = new ArrayList<>(operands.size()); + for (RexExpression operand : operands) { + rewritten.add(toRowExpression(operand)); + } + return new RexExpression.FunctionCall(call.getDataType(), call.getFunctionName(), rewritten, call.isDistinct(), + call.isIgnoreNulls()); + } + + private static void collectSymbolOrdinals(RexExpression expression, IntSet symbolOrdinals) { + if (expression instanceof RexExpression.PatternFieldRef) { + symbolOrdinals.add(((RexExpression.PatternFieldRef) expression).getSymbolOrdinal()); + } else if (expression instanceof RexExpression.FunctionCall) { + for (RexExpression operand : ((RexExpression.FunctionCall) expression).getFunctionOperands()) { + collectSymbolOrdinals(operand, symbolOrdinals); + } + } + } + + private static boolean containsPatternFieldRef(RexExpression expression) { + if (expression instanceof RexExpression.PatternFieldRef) { + return true; + } + if (expression instanceof RexExpression.FunctionCall) { + for (RexExpression operand : ((RexExpression.FunctionCall) expression).getFunctionOperands()) { + if (containsPatternFieldRef(operand)) { + return true; + } + } + } + return false; + } + + private static boolean containsMatchFunction(RexExpression expression) { + if (!(expression instanceof RexExpression.FunctionCall)) { + return false; + } + RexExpression.FunctionCall call = (RexExpression.FunctionCall) expression; + if (MATCH_FUNCTIONS.contains(call.getFunctionName())) { + return true; + } + for (RexExpression operand : call.getFunctionOperands()) { + if (containsMatchFunction(operand)) { + return true; + } + } + return false; + } + + private static RexExpression singleOperand(RexExpression.FunctionCall call) { + List operands = call.getFunctionOperands(); + if (operands.isEmpty()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Expecting an operand in MATCH_RECOGNIZE expression: '" + call + "'."); + } + return operands.get(0); + } + + /// Type of the column a bare pattern field reference reads. Unlike a navigation call, a bare reference carries no + /// declared type of its own. + private static ColumnDataType columnType(DataSchema inputSchema, RexExpression.PatternFieldRef ref) { + int index = ref.getIndex(); + if (index < 0 || index >= inputSchema.size()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE pattern field reference '" + ref + "' is out of range for input schema " + inputSchema); + } + return inputSchema.getColumnDataType(index); + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchLimits.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchLimits.java new file mode 100644 index 000000000000..cc55301a0b0e --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchLimits.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; +import org.apache.pinot.query.planner.plannode.PlanNode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; + + +/// Resolution of the two MATCH_RECOGNIZE resource limits. +/// +/// ## Both limits throw, they never truncate +/// +/// Pattern recognition has no meaningful partial answer: dropping a match, or cutting a match short, returns rows that +/// look plausible but are wrong, and nothing in the response says so. Unlike the window operator, which offers a +/// `BREAK` overflow mode, both limits here can only raise an error. +/// +/// ## Resolution order +/// +/// Highest precedence first, mirroring `maxRowsInWindow`: +/// 1. the per node hint, read from the `matchOptions` hint of the plan node; +/// 2. the query option, e.g. `SET maxRowsInMatchPartition = 50000`; +/// 3. the server cluster config, e.g. `pinot.query.match.max.rows.per.partition`, which `QueryRunner` folds into the +/// op chain metadata under the query option key when the query did not set one; +/// 4. the default declared here. +/// +/// The hint tier exists so a planner rule can pin a limit on an individual MATCH_RECOGNIZE node; no SQL syntax +/// attaches such a hint yet, so in practice the query option is the highest tier a user can reach today. +public final class MatchLimits { + private MatchLimits() { + } + + /// Hint namespace on the MATCH_RECOGNIZE plan node. + public static final String MATCH_HINT_OPTIONS = "matchOptions"; + /// Hint key for [QueryOptionKey#MAX_ROWS_IN_MATCH_PARTITION]. + public static final String MAX_ROWS_IN_MATCH_PARTITION_HINT = "max_rows_in_match_partition"; + /// Hint key for [QueryOptionKey#MAX_STEPS_PER_MATCH_ATTEMPT]. + public static final String MAX_STEPS_PER_MATCH_ATTEMPT_HINT = "max_steps_per_match_attempt"; + + /// Server config key backing [QueryOptionKey#MAX_ROWS_IN_MATCH_PARTITION]. + public static final String KEY_OF_MAX_ROWS_IN_MATCH_PARTITION = "pinot.query.match.max.rows.per.partition"; + /// Server config key backing [QueryOptionKey#MAX_STEPS_PER_MATCH_ATTEMPT]. + public static final String KEY_OF_MAX_STEPS_PER_MATCH_ATTEMPT = "pinot.query.match.max.steps.per.attempt"; + + public static final int DEFAULT_MAX_ROWS_IN_MATCH_PARTITION = 1_000_000; + /// A linear, non-backtracking match costs a small constant number of automaton transitions per row - measured at 2 + /// for `(A A)+`, 3 for `A+` and 5 for `(A|B)+` - so this budget has to dominate that constant + /// times [#DEFAULT_MAX_ROWS_IN_MATCH_PARTITION], or a partition that the row limit explicitly admits would be + /// rejected as if its PATTERN were ambiguous. At the previous value of one million steps, `PATTERN (A+)` could not + /// span a partition of more than 333,332 rows even though [#DEFAULT_MAX_ROWS_IN_MATCH_PARTITION] sanctions three + /// times that. + /// The factor of 16 leaves headroom for realistic linear patterns while still capping a catastrophically + /// backtracking one well below a query timeout: 16M transitions is a fraction of a second of matcher work, so + /// cancellation latency is unchanged in practice. + public static final long DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT = 16L * DEFAULT_MAX_ROWS_IN_MATCH_PARTITION; + + public static int getMaxRowsInMatchPartition(Map opChainMetadata, PlanNode.NodeHint nodeHint) { + String hintValue = resolveHint(nodeHint, MAX_ROWS_IN_MATCH_PARTITION_HINT); + if (hintValue != null) { + return parsePositiveInt(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, hintValue); + } + Integer optionValue = QueryOptionsUtils.getMaxRowsInMatchPartition(opChainMetadata); + return optionValue != null ? optionValue : DEFAULT_MAX_ROWS_IN_MATCH_PARTITION; + } + + public static long getMaxStepsPerMatchAttempt(Map opChainMetadata, PlanNode.NodeHint nodeHint) { + String hintValue = resolveHint(nodeHint, MAX_STEPS_PER_MATCH_ATTEMPT_HINT); + if (hintValue != null) { + return parsePositiveLong(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, hintValue); + } + Long optionValue = QueryOptionsUtils.getMaxStepsPerMatchAttempt(opChainMetadata); + return optionValue != null ? optionValue : DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT; + } + + @Nullable + private static String resolveHint(PlanNode.NodeHint nodeHint, String hintKey) { + Map matchOptions = nodeHint.getHintOptions().get(MATCH_HINT_OPTIONS); + if (matchOptions != null) { + String hintValue = matchOptions.get(hintKey); + if (hintValue != null) { + return hintValue; + } + } + return null; + } + + private static int parsePositiveInt(String name, String value) { + return (int) parsePositiveLong(name, value, Integer.MAX_VALUE); + } + + private static long parsePositiveLong(String name, String value) { + return parsePositiveLong(name, value, Long.MAX_VALUE); + } + + private static long parsePositiveLong(String name, String value, long maxValue) { + long parsed; + try { + parsed = Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(name + " must be a positive integer, got: '" + value + "'"); + } + if (parsed <= 0 || parsed > maxValue) { + throw new IllegalArgumentException( + name + " must be a positive integer no larger than " + maxValue + ", got: " + parsed); + } + return parsed; + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTape.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTape.java new file mode 100644 index 000000000000..7d512ff9e630 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTape.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.planner.plannode.PatternSymbol; + + +/// The classifier tape of the match currently being explored: which pattern variable each row of the candidate match +/// is mapped to, plus everything the MEASURES and DEFINE expressions need in order to navigate it. +/// +/// ## Append only, O(1) backtrack +/// +/// The tape grows by one entry every time the matcher consumes a row ([#push]) and shrinks by one every time it +/// backtracks over that row ([#pop]). Both are O(1): [#pop] only decrements lengths, it never rebuilds an +/// index. The per symbol position lists are maintained the same way, which makes [#lastRow] and +/// [#firstRow] O(1) as well, so logical navigation does not degrade into a scan of the match. +/// +/// ## Rows are addressed by partition index +/// +/// All row indexes handed out by this class are indexes into the partition, not into the match, because physical +/// navigation (`PREV` / `NEXT`) may leave the match while staying inside the partition. The invariant +/// `getEndPos() == getStartPos() + getLength()` always holds: the rows of a match are contiguous. +/// +/// ## Running semantics +/// +/// While the matcher is exploring, the tape holds only the rows matched so far, so a DEFINE predicate evaluated +/// through it automatically sees SQL:2016 *running* semantics. The candidate row is pushed *before* its own +/// predicate is evaluated, which is what makes `PREV(A.col, 0)` inside `DEFINE A` resolve to the current +/// row. Once the match is complete the tape holds all of its rows, so the same lookups give *final* semantics for +/// MEASURES. +/// +/// Not thread safe: one instance belongs to one [PartitionMatcher]. +public class MatchTape { + /// Returned by [#lastRow] and [#firstRow] when no such row exists. + public static final int NO_ROW = -1; + + private final List _patternSymbols; + /// Partition row indexes mapped to each symbol, in ascending order; index is the symbol ordinal. + private final IntArrayList[] _rowsBySymbol; + + private List _partitionRows = List.of(); + private int _startPos; + private int[] _labels = new int[16]; + private int _length; + private long _matchNumber; + + public MatchTape(List patternSymbols) { + _patternSymbols = patternSymbols; + _rowsBySymbol = new IntArrayList[patternSymbols.size()]; + for (int i = 0; i < _rowsBySymbol.length; i++) { + _rowsBySymbol[i] = new IntArrayList(); + } + } + + /// Starts a new match attempt at `startPos` of `partitionRows`. `matchNumber` is the value + /// `MATCH_NUMBER()` reports, which SQL:2016 assigns before the match is known to succeed. + public void reset(List partitionRows, int startPos, long matchNumber) { + _partitionRows = partitionRows; + _startPos = startPos; + _length = 0; + _matchNumber = matchNumber; + for (IntArrayList rows : _rowsBySymbol) { + rows.clear(); + } + } + + /// Maps the row at [#getEndPos()] to `symbolOrdinal` and extends the tape by one row. + public void push(int symbolOrdinal) { + if (_length == _labels.length) { + int[] grown = new int[_labels.length * 2]; + System.arraycopy(_labels, 0, grown, 0, _length); + _labels = grown; + } + _labels[_length] = symbolOrdinal; + _rowsBySymbol[symbolOrdinal].add(_startPos + _length); + _length++; + } + + /// Removes the last row from the tape. O(1); this is what makes backtracking cheap. + public void pop() { + _length--; + IntArrayList rows = _rowsBySymbol[_labels[_length]]; + rows.removeInt(rows.size() - 1); + } + + public List getPartitionRows() { + return _partitionRows; + } + + /// Partition index of the first row of the match. + public int getStartPos() { + return _startPos; + } + + /// Partition index one past the last row of the match, i.e. where the next row would be consumed. + public int getEndPos() { + return _startPos + _length; + } + + /// Number of rows currently on the tape. + public int getLength() { + return _length; + } + + public long getMatchNumber() { + return _matchNumber; + } + + /// Symbol ordinal the row at partition index `rowIndex` is mapped to, or [#NO_ROW] if that row is not + /// part of the match. + public int labelAt(int rowIndex) { + int offset = rowIndex - _startPos; + return offset >= 0 && offset < _length ? _labels[offset] : NO_ROW; + } + + /// Name of the pattern variable the row at partition index `rowIndex` is mapped to, i.e. the value of + /// `CLASSIFIER()` at that row, or `null` if that row is not part of the match. + @Nullable + public String classifierAt(int rowIndex) { + int label = labelAt(rowIndex); + return label == NO_ROW ? null : _patternSymbols.get(label).getName(); + } + + /// Partition index of the `(offset + 1)`-th row from the **end** of the rows mapped to + /// `symbolOrdinal`, i.e. the row `LAST(symbol.col, offset)` designates. Pass + /// [RexExpression.PatternFieldRef#UNIVERSAL_SYMBOL_ORDINAL] to navigate the rows of the whole match instead of + /// a single variable. Returns [#NO_ROW] if there are not that many such rows. + public int lastRow(int symbolOrdinal, int offset) { + if (symbolOrdinal == RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + int index = _length - 1 - offset; + return index >= 0 ? _startPos + index : NO_ROW; + } + IntArrayList rows = _rowsBySymbol[symbolOrdinal]; + int index = rows.size() - 1 - offset; + return index >= 0 ? rows.getInt(index) : NO_ROW; + } + + /// Partition index of the `(offset + 1)`-th row from the **start** of the rows mapped to + /// `symbolOrdinal`, i.e. the row `FIRST(symbol.col, offset)` designates. Behaves like [#lastRow] + /// otherwise. + public int firstRow(int symbolOrdinal, int offset) { + if (symbolOrdinal == RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + return offset < _length ? _startPos + offset : NO_ROW; + } + IntArrayList rows = _rowsBySymbol[symbolOrdinal]; + return offset < rows.size() ? rows.getInt(offset) : NO_ROW; + } + + /// Partition indexes of the rows mapped to `symbolOrdinal`, in ascending order, or of the whole match for + /// [RexExpression.PatternFieldRef#UNIVERSAL_SYMBOL_ORDINAL]. Used by the MEASURES aggregates. The returned + /// list is the live backing list and is invalidated by the next [#push] or [#pop]. + public IntArrayList rowsOf(int symbolOrdinal) { + if (symbolOrdinal == RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + IntArrayList allRows = new IntArrayList(_length); + for (int i = 0; i < _length; i++) { + allRows.add(_startPos + i); + } + return allRows; + } + return _rowsBySymbol[symbolOrdinal]; + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTerm.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTerm.java new file mode 100644 index 000000000000..704e310bb2b1 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/MatchTerm.java @@ -0,0 +1,383 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.runtime.operator.operands.TransformOperand; +import org.apache.pinot.query.runtime.operator.utils.TypeUtils; +import org.apache.pinot.spi.exception.QueryErrorCode; + + +/// A leaf of a MEASURES or DEFINE expression whose value depends on the match rather than on a single row: a row +/// pattern navigation, `CLASSIFIER()`, `MATCH_NUMBER()`, or a single variable aggregate. +/// +/// [MatchExpression] replaces every such leaf with a slot in a synthetic row, so that everything above the +/// leaves - comparisons, boolean connectives, arithmetic, scalar functions - is evaluated by Pinot's ordinary +/// [TransformOperand] machinery instead of a second expression interpreter. +/// +/// Implementations are stateless with respect to the match: all match state is passed in through the +/// [MatchTape], so one instance is reused across every match of every partition. +public interface MatchTerm { + + /// The type this term reports to the enclosing expression. Values returned by [#evaluate] are in the + /// corresponding [stored][ColumnDataType#getStoredType()] representation, exactly like the values of a real + /// input row. + ColumnDataType getResultType(); + + @Nullable + Object evaluate(MatchTape tape); + + /// A row pattern navigation: an optional logical step (`FIRST` / `LAST`) that designates a row of the + /// match, followed by an optional physical step (`PREV` / `NEXT`) that moves a fixed number of rows + /// relative to it, and finally a column read. + /// + /// Both steps are needed because SQL:2016 nests them, e.g. `PREV(LAST(A.price), 2)` designates the last row + /// mapped to `A` and then moves two rows back. The logical step is bounded by the match; the physical step is + /// bounded only by the partition, so it may legally read a row outside the match. Either step falling off its bound + /// yields `null`, as the standard requires. + final class Navigation implements MatchTerm { + private final int _symbolOrdinal; + private final boolean _fromEnd; + private final int _logicalOffset; + private final int _physicalDelta; + private final int _columnIndex; + private final ColumnDataType _resultType; + private final ColumnDataType _storedType; + private final boolean _requiresConversion; + + /// @param symbolOrdinal pattern variable to navigate, or + /// [org.apache.pinot.query.planner.logical.RexExpression.PatternFieldRef#UNIVERSAL_SYMBOL_ORDINAL] + /// for an unqualified column reference, which navigates the rows of the whole match + /// @param fromEnd `true` for `LAST`, `false` for `FIRST` + /// @param logicalOffset how many rows back from the end (or forward from the start) of the designated variable + /// @param physicalDelta rows to move in the partition afterwards; negative for `PREV`, positive for + /// `NEXT`, zero when there is no physical step + /// @param columnIndex column to read, as an index into the input row of the MATCH_RECOGNIZE node + /// @param sourceType type of that input column; values already use its stored representation + public Navigation(int symbolOrdinal, boolean fromEnd, int logicalOffset, int physicalDelta, int columnIndex, + ColumnDataType sourceType, ColumnDataType resultType) { + _symbolOrdinal = symbolOrdinal; + _fromEnd = fromEnd; + _logicalOffset = logicalOffset; + _physicalDelta = physicalDelta; + _columnIndex = columnIndex; + _resultType = resultType; + _storedType = resultType.getStoredType(); + _requiresConversion = sourceType.getStoredType() != _storedType; + } + + @Override + public ColumnDataType getResultType() { + return _resultType; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + int rowIndex = _fromEnd ? tape.lastRow(_symbolOrdinal, _logicalOffset) + : tape.firstRow(_symbolOrdinal, _logicalOffset); + if (rowIndex == MatchTape.NO_ROW) { + return null; + } + long physicalRowIndex = (long) rowIndex + _physicalDelta; + List rows = tape.getPartitionRows(); + if (physicalRowIndex < 0 || physicalRowIndex >= rows.size()) { + return null; + } + Object value = rows.get((int) physicalRowIndex)[_columnIndex]; + // The declared type of the navigation may be wider than the column's (e.g. BIG_DECIMAL for a DOUBLE column), + // and the enclosing operand compares against that declared type. + return value != null && _requiresConversion ? TypeUtils.convert(value, _storedType) : value; + } + } + + /// `CLASSIFIER()`: the name of the pattern variable the designated row is mapped to. With ONE ROW PER MATCH + /// the designated row is the last row of the match, which is also the current row while a DEFINE predicate is being + /// evaluated. + final class Classifier implements MatchTerm { + public static final Classifier INSTANCE = new Classifier(); + + private Classifier() { + } + + @Override + public ColumnDataType getResultType() { + return ColumnDataType.STRING; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + return tape.classifierAt(tape.getEndPos() - 1); + } + } + + /// `MATCH_NUMBER()`: the sequential number of the match within its partition, starting at 1. + final class MatchNumber implements MatchTerm { + public static final MatchNumber INSTANCE = new MatchNumber(); + + private MatchNumber() { + } + + @Override + public ColumnDataType getResultType() { + return ColumnDataType.LONG; + } + + @Override + public Object evaluate(MatchTape tape) { + return tape.getMatchNumber(); + } + } + + /// A single variable aggregate in MEASURES, e.g. `SUM(A.price)` or `COUNT(*)`: the aggregate of an + /// expression evaluated over every row of the match that is mapped to one pattern variable. + /// + /// The argument is evaluated by an ordinary [TransformOperand] against the raw input row, so any scalar + /// expression works, e.g. `SUM(A.price * A.quantity)`. Nulls are skipped, as SQL requires; an aggregate over + /// zero rows is `null` except for `COUNT`, which is `0`. + final class Aggregate implements MatchTerm { + private final Kind _kind; + private final int _symbolOrdinal; + @Nullable + private final TransformOperand _argument; + private final ColumnDataType _resultType; + private final ColumnDataType _storedType; + private final boolean _requiresResultConversion; + + /// @param argument the aggregated expression evaluated against an input row, or `null` for `COUNT(*)`, + /// which counts rows rather than values + public Aggregate(Kind kind, int symbolOrdinal, @Nullable TransformOperand argument, ColumnDataType resultType) { + _kind = kind; + _symbolOrdinal = symbolOrdinal; + _argument = argument; + _resultType = resultType; + _storedType = resultType.getStoredType(); + if (argument != null && argument.getResultType().isArray()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Multi-value operand type '" + argument.getResultType() + "' is not supported for " + kind + + " in a MATCH_RECOGNIZE MEASURES clause. Reduce the array to a scalar before aggregating it."); + } + if ((kind == Kind.SUM || kind == Kind.AVG) && !_storedType.isNumber()) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE " + kind + " requires a numeric result type, got: " + resultType + "."); + } + _requiresResultConversion = argument != null && argument.getResultType().getStoredType() != _storedType; + } + + @Override + public ColumnDataType getResultType() { + return _resultType; + } + + @Nullable + @Override + public Object evaluate(MatchTape tape) { + if (_kind == Kind.COUNT && _argument == null + && _symbolOrdinal == RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL) { + return (long) tape.getLength(); + } + IntArrayList rows = tape.rowsOf(_symbolOrdinal); + if (_kind == Kind.COUNT && _argument == null) { + return (long) rows.size(); + } + List partitionRows = tape.getPartitionRows(); + switch (_kind) { + case COUNT: + return countNonNull(rows, partitionRows); + case MIN: + case MAX: + return evaluateExtremum(rows, partitionRows); + case SUM: + case AVG: + return evaluateSumOrAverage(rows, partitionRows); + default: + throw new IllegalStateException("Unexpected MATCH_RECOGNIZE aggregate: " + _kind); + } + } + + private long countNonNull(IntArrayList rows, List partitionRows) { + long count = 0; + for (int i = 0; i < rows.size(); i++) { + Object value = _argument.apply(partitionRows.get(rows.getInt(i))); + if (value != null) { + count++; + } + } + return count; + } + + @Nullable + private Object evaluateExtremum(IntArrayList rows, List partitionRows) { + Object extremum = null; + for (int i = 0; i < rows.size(); i++) { + Object value = _argument.apply(partitionRows.get(rows.getInt(i))); + if (value == null) { + continue; + } + if (extremum == null || (_kind == Kind.MIN ? compare(value, extremum) < 0 : compare(value, extremum) > 0)) { + extremum = value; + } + } + if (extremum == null || !_requiresResultConversion) { + return extremum; + } + return TypeUtils.convert(extremum, _storedType); + } + + @Nullable + private Object evaluateSumOrAverage(IntArrayList rows, List partitionRows) { + switch (_storedType) { + case INT: + case LONG: + return evaluateIntegralSumOrAverage(rows, partitionRows); + case FLOAT: + case DOUBLE: + return evaluateFloatingPointSumOrAverage(rows, partitionRows); + case BIG_DECIMAL: + return evaluateDecimalSumOrAverage(rows, partitionRows); + default: + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE " + _kind + " requires a numeric result type, got: " + _resultType + "."); + } + } + + @Nullable + private Object evaluateIntegralSumOrAverage(IntArrayList rows, List partitionRows) { + long sum = 0; + long count = 0; + for (int i = 0; i < rows.size(); i++) { + Object value = _argument.apply(partitionRows.get(rows.getInt(i))); + if (value != null) { + sum += numericValue(value).longValue(); + count++; + } + } + if (count == 0) { + return null; + } + long result = _kind == Kind.AVG ? sum / count : sum; + if (_storedType == ColumnDataType.INT) { + return (int) result; + } + return result; + } + + @Nullable + private Object evaluateFloatingPointSumOrAverage(IntArrayList rows, List partitionRows) { + double sum = 0; + long count = 0; + for (int i = 0; i < rows.size(); i++) { + Object value = _argument.apply(partitionRows.get(rows.getInt(i))); + if (value != null) { + sum += numericValue(value).doubleValue(); + count++; + } + } + if (count == 0) { + return null; + } + double result = _kind == Kind.AVG ? sum / count : sum; + if (_storedType == ColumnDataType.FLOAT) { + return (float) result; + } + return result; + } + + @Nullable + private Object evaluateDecimalSumOrAverage(IntArrayList rows, List partitionRows) { + BigDecimal sum = BigDecimal.ZERO; + long count = 0; + for (int i = 0; i < rows.size(); i++) { + Object value = _argument.apply(partitionRows.get(rows.getInt(i))); + if (value != null) { + sum = sum.add(toBigDecimal(numericValue(value))); + count++; + } + } + if (count == 0) { + return null; + } + return _kind == Kind.AVG ? sum.divide(BigDecimal.valueOf(count), MathContext.DECIMAL128) : sum; + } + + private Number numericValue(Object value) { + if (value instanceof Number) { + return (Number) value; + } + throw QueryErrorCode.QUERY_EXECUTION.asException( + "MATCH_RECOGNIZE " + _kind + " requires scalar numeric values, got: " + value.getClass().getName() + "."); + } + + private static BigDecimal toBigDecimal(Number value) { + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } + if (value instanceof BigInteger) { + return new BigDecimal((BigInteger) value); + } + if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) { + return BigDecimal.valueOf(value.longValue()); + } + return BigDecimal.valueOf(value.doubleValue()); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static int compare(Object left, Object right) { + return ((Comparable) left).compareTo(right); + } + + /// The aggregate functions supported inside MEASURES. Anything else is rejected at operator construction time + /// rather than silently dropped. + public enum Kind { + COUNT, SUM, MIN, MAX, AVG; + + /// Resolves `functionName` to a supported aggregate, or `null` if it is not an aggregate at all. + @Nullable + public static Kind of(String functionName) { + for (Kind kind : values()) { + if (kind.name().equals(functionName)) { + return kind; + } + } + return null; + } + + /// Throws with an actionable message for an aggregate that exists in Pinot but is not supported inside + /// MEASURES yet. + public static Kind resolveOrThrow(String functionName) { + Kind kind = of(functionName); + if (kind == null) { + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Aggregate function '" + functionName + "' is not supported in a MATCH_RECOGNIZE MEASURES clause. " + + "Supported aggregates are " + Arrays.toString(values()) + "."); + } + return kind; + } + } + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcher.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcher.java new file mode 100644 index 000000000000..8ec7fcdf2d29 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcher.java @@ -0,0 +1,373 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import javax.annotation.Nullable; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.Transition; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; + + +/// Runs a [PatternNfa] over the rows of one partition and reports the SQL:2016 preferred match starting at a +/// given row. +/// +/// ## First match found is the preferred match +/// +/// The search is a depth first exploration that always takes the first not-yet-tried transition of the current state. +/// [PatternToNfaCompiler] orders the transitions of every state by SQL:2016 preference, so this enumeration is +/// exactly the preferment order and [#match] can return as soon as it reaches the accepting state. No candidate +/// match is ever scored or compared against another. +/// +/// ## Only choice points are retained +/// +/// Deterministic states are followed directly rather than pushed onto the search stack. The stack contains only +/// states with more than one outgoing transition, together with a checkpoint into a separate primitive counter-undo +/// log. Backtracking truncates the classifier tape to the choice point's row and restores that log. For a linear +/// `A+` match this retains one compact choice point per row rather than three full frames per row. The combined +/// primitive-array payload is hard-capped so raising the transition-step option cannot permit unbounded retained +/// backtracking state. +/// +/// ## Running semantics come for free +/// +/// A candidate row is pushed onto the tape *before* its DEFINE predicate is evaluated, so the predicate sees the +/// candidate as the current row and every navigation resolves against the rows matched so far. That is precisely the +/// SQL:2016 running semantics of DEFINE. If the predicate fails, the row is popped again. +/// +/// Not thread safe: one instance owns one tape and one search stack. +public class PartitionMatcher { + /// Returned by [#match] when no match starts at the requested row. + public static final int NO_MATCH = -1; + + private static final int NO_POS = -1; + private static final int INITIAL_STACK_CAPACITY = 64; + private static final int TERMINATION_CHECK_INTERVAL = 1024; + private static final long MAX_RETAINED_BACKTRACKING_BYTES = 64L * 1024 * 1024; + private static final Runnable NO_OP_TERMINATION_CHECKER = () -> { }; + + private final PatternNfa _nfa; + /// DEFINE predicate per symbol ordinal; `null` means the variable matches every row, per SQL:2016. + private final MatchExpression[] _definitions; + private final MatchTape _tape; + private final long _maxStepsPerMatchAttempt; + private final long _maxRetainedBacktrackingBytes; + private final Runnable _terminationChecker; + private final int[] _counters; + private final int[] _loopStartPos; + + private int[] _choiceState = new int[INITIAL_STACK_CAPACITY]; + private int[] _choicePos = new int[INITIAL_STACK_CAPACITY]; + private int[] _choiceNextTransition = new int[INITIAL_STACK_CAPACITY]; + private int[] _choiceCounterUndoSize = new int[INITIAL_STACK_CAPACITY]; + private int _choiceStackSize; + + private int[] _undoCounter = new int[INITIAL_STACK_CAPACITY]; + private int[] _undoCount = new int[INITIAL_STACK_CAPACITY]; + private int[] _undoLoopStart = new int[INITIAL_STACK_CAPACITY]; + private int _counterUndoSize; + + private int _state; + private int _pos; + private int _partitionSize; + private int _matchStartPos; + private long _steps; + + public PartitionMatcher(PatternNfa nfa, List patternSymbols, DataSchema inputSchema, + long maxStepsPerMatchAttempt) { + this(nfa, patternSymbols, inputSchema, maxStepsPerMatchAttempt, NO_OP_TERMINATION_CHECKER, + MAX_RETAINED_BACKTRACKING_BYTES); + } + + public PartitionMatcher(PatternNfa nfa, List patternSymbols, DataSchema inputSchema, + long maxStepsPerMatchAttempt, Runnable terminationChecker) { + this(nfa, patternSymbols, inputSchema, maxStepsPerMatchAttempt, terminationChecker, + MAX_RETAINED_BACKTRACKING_BYTES); + } + + @VisibleForTesting + PartitionMatcher(PatternNfa nfa, List patternSymbols, DataSchema inputSchema, + long maxStepsPerMatchAttempt, Runnable terminationChecker, long maxRetainedBacktrackingBytes) { + _nfa = nfa; + _tape = new MatchTape(patternSymbols); + _maxStepsPerMatchAttempt = maxStepsPerMatchAttempt; + if (maxRetainedBacktrackingBytes < getRetainedBacktrackingBytes()) { + throw new IllegalArgumentException("The retained backtracking byte limit must fit the initial matcher state"); + } + _maxRetainedBacktrackingBytes = maxRetainedBacktrackingBytes; + _terminationChecker = Objects.requireNonNull(terminationChecker, "terminationChecker"); + _definitions = new MatchExpression[patternSymbols.size()]; + for (int i = 0; i < _definitions.length; i++) { + // A pattern variable that appears in PATTERN without a DEFINE entry matches every row, per SQL:2016. + _definitions[i] = patternSymbols.get(i).getDefinition() == null ? null + : MatchExpression.compile(patternSymbols.get(i).getDefinition(), inputSchema); + } + _counters = new int[nfa.getNumCounters()]; + _loopStartPos = new int[nfa.getNumCounters()]; + } + + /// The classifier tape. After a successful [#match] it describes that match and stays valid until the next + /// call, so MEASURES are evaluated through it with final semantics. + public MatchTape getTape() { + return _tape; + } + + /// Releases the partition retained by the classifier tape after its final measures have been evaluated. + public void releasePartition() { + _tape.reset(List.of(), 0, 0); + _choiceStackSize = 0; + _counterUndoSize = 0; + } + + /// Finds the preferred match that starts exactly at `startPos`. + /// + /// @param matchNumber the value `MATCH_NUMBER()` reports; SQL:2016 assigns it before the match is known to + /// succeed, so it is passed in rather than derived + /// @return the partition index one past the last row of the match, which equals `startPos` for an empty match, + /// or [#NO_MATCH] if no match starts here + /// @throws org.apache.pinot.spi.exception.QueryException if the attempt exceeds the configured step budget. It + /// throws rather than giving up, because giving up would silently drop matches. + public int match(List partitionRows, int startPos, long matchNumber) { + _tape.reset(partitionRows, startPos, matchNumber); + Arrays.fill(_counters, 0); + Arrays.fill(_loopStartPos, NO_POS); + _choiceStackSize = 0; + _counterUndoSize = 0; + _state = _nfa.getStartState(); + _pos = startPos; + _partitionSize = partitionRows.size(); + _matchStartPos = startPos; + _steps = 0; + + int acceptState = _nfa.getAcceptState(); + while (true) { + if (_state == acceptState) { + return _pos; + } + + List transitions = _nfa.getState(_state).getTransitions(); + if (transitions.size() > 1) { + pushChoice(_state, _pos); + if (takeNextChoice()) { + continue; + } + } else if (transitions.size() == 1) { + recordStep(_pos); + if (tryApply(transitions.get(0), _pos)) { + continue; + } + } + + if (takeNextChoice()) { + continue; + } + restoreTo(startPos, 0); + return NO_MATCH; + } + } + + /// Applies `transition` at `pos`, updating the current state when its guard holds. + /// + /// @return whether the transition was taken; on success [#_state] and [#_pos] hold its target + private boolean tryApply(Transition transition, int pos) { + int target = transition.getTarget(); + switch (transition.getKind()) { + case MATCH: { + if (pos >= _partitionSize) { + return false; + } + int symbolOrdinal = transition.getOperand(); + // Push before evaluating so the predicate sees the candidate row as the current row. + _tape.push(symbolOrdinal); + MatchExpression definition = _definitions[symbolOrdinal]; + if (definition != null && !definition.test(_tape)) { + _tape.pop(); + return false; + } + return advanceTo(target, pos + 1); + } + case EPSILON: + return advanceTo(target, pos); + case START_LOOP: { + int counterId = transition.getOperand(); + pushCounterUndo(counterId); + _counters[counterId] = 0; + _loopStartPos[counterId] = NO_POS; + return advanceTo(target, pos); + } + case REPEAT: { + int counterId = transition.getOperand(); + int maxRepeat = transition.getBound(); + if (maxRepeat != PatternNfa.UNBOUNDED && _counters[counterId] >= maxRepeat) { + return false; + } + // Empty cycle guard: the previous iteration of this quantifier consumed no row, so another one never will. + if (_loopStartPos[counterId] == pos) { + return false; + } + pushCounterUndo(counterId); + _counters[counterId]++; + _loopStartPos[counterId] = pos; + return advanceTo(target, pos); + } + case EXIT_LOOP: { + int counterId = transition.getOperand(); + // An empty iteration may be repeated vacuously any number of times, so it satisfies any minimum. + if (_counters[counterId] < transition.getBound() && _loopStartPos[counterId] != pos) { + return false; + } + return advanceTo(target, pos); + } + case ANCHOR_START: + if (pos != 0) { + return false; + } + return advanceTo(target, pos); + case ANCHOR_END: + if (pos != _partitionSize) { + return false; + } + return advanceTo(target, pos); + default: + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported MATCH_RECOGNIZE pattern transition: " + transition.getKind()); + } + } + + private boolean advanceTo(int state, int pos) { + _state = state; + _pos = pos; + return true; + } + + private void pushChoice(int state, int pos) { + if (_choiceStackSize == _choiceState.length) { + growChoiceStack(); + } + _choiceState[_choiceStackSize] = state; + _choicePos[_choiceStackSize] = pos; + _choiceNextTransition[_choiceStackSize] = 0; + _choiceCounterUndoSize[_choiceStackSize] = _counterUndoSize; + _choiceStackSize++; + } + + /// Restores successive choice points and takes their next valid transition, in preference order. + private boolean takeNextChoice() { + while (_choiceStackSize > 0) { + int top = _choiceStackSize - 1; + int choicePos = _choicePos[top]; + int undoSize = _choiceCounterUndoSize[top]; + restoreTo(choicePos, undoSize); + List transitions = _nfa.getState(_choiceState[top]).getTransitions(); + while (_choiceNextTransition[top] < transitions.size()) { + Transition transition = transitions.get(_choiceNextTransition[top]++); + recordStep(choicePos); + if (tryApply(transition, choicePos)) { + return true; + } + restoreTo(choicePos, undoSize); + } + _choiceStackSize--; + } + return false; + } + + private void pushCounterUndo(int counterId) { + if (_counterUndoSize == _undoCounter.length) { + growCounterUndoLog(); + } + _undoCounter[_counterUndoSize] = counterId; + _undoCount[_counterUndoSize] = _counters[counterId]; + _undoLoopStart[_counterUndoSize] = _loopStartPos[counterId]; + _counterUndoSize++; + } + + private void restoreTo(int pos, int undoSize) { + while (_tape.getEndPos() > pos) { + _tape.pop(); + } + while (_counterUndoSize > undoSize) { + int undo = --_counterUndoSize; + int counterId = _undoCounter[undo]; + _counters[counterId] = _undoCount[undo]; + _loopStartPos[counterId] = _undoLoopStart[undo]; + } + } + + private void recordStep(int pos) { + if (++_steps > _maxStepsPerMatchAttempt) { + // The step count includes the transitions that make linear progress, so a long match over a large partition can + // hit this without any backtracking at all. The partition and consumed row counts distinguish the two cases. + throw QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.asException( + "MATCH_RECOGNIZE exceeded the maximum of " + _maxStepsPerMatchAttempt + + " pattern matching steps for the match attempt starting at row " + _matchStartPos + " of a " + + _partitionSize + "-row partition (" + (pos - _matchStartPos) + " rows consumed so far). Raise the '" + + QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT + "' query option, or if the row count consumed is far " + + "below the step count, make the PATTERN less ambiguous and tighten the DEFINE predicates."); + } + if ((_steps & (TERMINATION_CHECK_INTERVAL - 1)) == 0) { + _terminationChecker.run(); + } + } + + private void growChoiceStack() { + int capacity = _choiceState.length * 2; + checkRetainedBacktrackingCapacity(capacity, _undoCounter.length); + _choiceState = Arrays.copyOf(_choiceState, capacity); + _choicePos = Arrays.copyOf(_choicePos, capacity); + _choiceNextTransition = Arrays.copyOf(_choiceNextTransition, capacity); + _choiceCounterUndoSize = Arrays.copyOf(_choiceCounterUndoSize, capacity); + } + + private void growCounterUndoLog() { + int capacity = _undoCounter.length * 2; + checkRetainedBacktrackingCapacity(_choiceState.length, capacity); + _undoCounter = Arrays.copyOf(_undoCounter, capacity); + _undoCount = Arrays.copyOf(_undoCount, capacity); + _undoLoopStart = Arrays.copyOf(_undoLoopStart, capacity); + } + + @VisibleForTesting + long getRetainedBacktrackingBytes() { + return retainedBacktrackingBytes(_choiceState.length, _undoCounter.length); + } + + private void checkRetainedBacktrackingCapacity(int choiceCapacity, int undoCapacity) { + long retainedBytes = retainedBacktrackingBytes(choiceCapacity, undoCapacity); + if (retainedBytes > _maxRetainedBacktrackingBytes) { + throw QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED.asException( + "MATCH_RECOGNIZE exceeded the hard limit of " + _maxRetainedBacktrackingBytes + + " bytes for retained pattern backtracking state. Reduce the partition size or simplify the PATTERN."); + } + } + + private static long retainedBacktrackingBytes(int choiceCapacity, int undoCapacity) { + return (long) Integer.BYTES * (4L * choiceCapacity + 3L * undoCapacity); + } + + /// The DEFINE predicate compiled for `symbolOrdinal`, or `null` if the variable matches every row. + @Nullable + MatchExpression getDefinition(int symbolOrdinal) { + return _definitions[symbolOrdinal]; + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternNfa.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternNfa.java new file mode 100644 index 000000000000..daa921aac223 --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternNfa.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.ArrayList; +import java.util.List; + + +/// A non-deterministic finite automaton compiled from a MATCH_RECOGNIZE `PATTERN` clause by +/// [PatternToNfaCompiler], and executed by [PartitionMatcher]. +/// +/// ## Prioritized transitions +/// +/// The transitions of a state are stored in **preference order**, highest preference first. A depth first search +/// that always tries the transitions of a state in list order therefore enumerates candidate matches in the SQL:2016 +/// "preferment order", and the first complete match it reaches is the preferred one. See +/// [PatternToNfaCompiler] for how each pattern construct maps onto that ordering. +/// +/// ## Counter registers +/// +/// Bounded quantifiers such as `A{2,5}` are **not** unrolled into repeated states. Every quantifier node +/// allocates one counter register; the loop is a single cycle in the automaton whose entry and exit edges are guarded +/// by that counter. The number of states is therefore linear in the size of the pattern text and independent of the +/// repetition bounds, so `A{1,10000}` costs the same to compile as `A+`. +/// +/// Instances are immutable and safe to share between the threads that execute different partitions. +public class PatternNfa { + /// Sentinel for [Transition#getBound()] of an unbounded [TransitionKind#REPEAT]. + public static final int UNBOUNDED = -1; + + private final List _states; + private final int _startState; + private final int _acceptState; + private final int _numCounters; + + PatternNfa(List states, int startState, int acceptState, int numCounters) { + List frozenStates = new ArrayList<>(states.size()); + for (State state : states) { + frozenStates.add(state.frozenCopy()); + } + _states = List.copyOf(frozenStates); + _startState = startState; + _acceptState = acceptState; + _numCounters = numCounters; + } + + public List getStates() { + return _states; + } + + public State getState(int stateId) { + return _states.get(stateId); + } + + public int getNumStates() { + return _states.size(); + } + + public int getStartState() { + return _startState; + } + + /// The single accepting state. Reaching it means the whole pattern has been matched. + public int getAcceptState() { + return _acceptState; + } + + /// Number of counter registers the automaton uses, i.e. the number of quantifiers in the pattern. + public int getNumCounters() { + return _numCounters; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("PatternNfa{start=").append(_startState).append(", accept=") + .append(_acceptState).append(", counters=").append(_numCounters).append('}'); + for (int stateId = 0; stateId < _states.size(); stateId++) { + builder.append('\n').append(stateId).append(':'); + for (Transition transition : _states.get(stateId).getTransitions()) { + builder.append(' ').append(transition); + } + } + return builder.toString(); + } + + /// What a transition does when it is taken. + public enum TransitionKind { + /// Consumes the current row if the DEFINE predicate of [Transition#getOperand()] holds for it. + MATCH, + /// Consumes nothing and has no side effect. + EPSILON, + /// Enters a quantifier: resets the counter register [Transition#getOperand()]. + START_LOOP, + /// Runs one more iteration of a quantifier. Allowed while the counter is below + /// [Transition#getBound()] repetitions and the previous iteration consumed at least one row; increments the + /// counter. + REPEAT, + /// Leaves a quantifier. Allowed once the counter reached [Transition#getBound()] repetitions, or once an + /// iteration turned out to be empty. + EXIT_LOOP, + /// The `^` anchor: allowed only at the first row of the partition. + ANCHOR_START, + /// The `$` anchor: allowed only past the last row of the partition. + ANCHOR_END + } + + /// One outgoing edge of a [State]. Its position in [State#getTransitions()] is its preference. + public static final class Transition { + private final TransitionKind _kind; + private final int _target; + private final int _operand; + private final int _bound; + + Transition(TransitionKind kind, int target, int operand, int bound) { + _kind = kind; + _target = target; + _operand = operand; + _bound = bound; + } + + public TransitionKind getKind() { + return _kind; + } + + /// The state this transition leads to. + public int getTarget() { + return _target; + } + + /// Pattern symbol ordinal for [TransitionKind#MATCH], counter register id for the loop kinds, unused + /// otherwise. + public int getOperand() { + return _operand; + } + + /// Maximum repetition count for [TransitionKind#REPEAT] ([#UNBOUNDED] if there is no upper bound), and + /// the minimum repetition count for [TransitionKind#EXIT_LOOP]. Unused otherwise. + public int getBound() { + return _bound; + } + + @Override + public String toString() { + switch (_kind) { + case MATCH: + return "MATCH(s" + _operand + ")->" + _target; + case EPSILON: + return "EPS->" + _target; + case START_LOOP: + return "START(c" + _operand + ")->" + _target; + case REPEAT: + return "REPEAT(c" + _operand + ",max=" + (_bound == UNBOUNDED ? "*" : _bound) + ")->" + _target; + case EXIT_LOOP: + return "EXIT(c" + _operand + ",min=" + _bound + ")->" + _target; + default: + return _kind + "->" + _target; + } + } + } + + /// A state of the automaton. Mutable only while [PatternToNfaCompiler] builds it. + public static final class State { + private final List _transitions; + + State() { + _transitions = new ArrayList<>(2); + } + + private State(List transitions) { + _transitions = List.copyOf(transitions); + } + + /// The outgoing transitions in preference order, highest preference first. + public List getTransitions() { + return _transitions; + } + + void addTransition(Transition transition) { + _transitions.add(transition); + } + + private State frozenCopy() { + return new State(_transitions); + } + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompiler.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompiler.java new file mode 100644 index 000000000000..f5941f47114f --- /dev/null +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompiler.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.ArrayList; +import java.util.List; +import org.apache.pinot.query.planner.plannode.RowPattern; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.State; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.Transition; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.TransitionKind; +import org.apache.pinot.spi.exception.QueryErrorCode; + + +/// Compiles a MATCH_RECOGNIZE `PATTERN` tree into a [PatternNfa] whose transitions are ordered by +/// preference. +/// +/// ## The invariant this class exists to establish +/// +/// **Every state lists its outgoing transitions in SQL:2016 preference order, so a depth first search that always +/// takes the first not-yet-tried transition of the current state enumerates candidate matches in preferment order, and +/// the *first* complete match it reaches is the preferred match.** [PartitionMatcher] relies on exactly +/// that: it never enumerates all matches and picks a best one, it stops at the first one. +/// +/// The three constructs that carry a preference are encoded as follows. +/// - **Alternation.** `A | B | C` emits one epsilon transition per branch, in source order, so the +/// leftmost branch that can complete the whole pattern wins. +/// - **Greedy quantifier.** The loop head lists the `REPEAT` edge before the `EXIT_LOOP` edge, so +/// one more repetition is preferred over leaving the loop. +/// - **Reluctant quantifier** (`*?`, `+?`, `??`, `{n,m}?`). The same two edges in the +/// opposite order, so leaving the loop is preferred over one more repetition. +/// +/// ## Quantifiers use counter registers, never state unrolling +/// +/// A quantifier compiles to a single cycle: +/// ``` +/// from --START_LOOP(c)--> loopHead +/// loopHead --REPEAT(c, max)--> bodyStart ... bodyEnd --EPSILON--> loopHead +/// loopHead --EXIT_LOOP(c, min)--> exit +/// ``` +/// `START_LOOP` resets register `c`, `REPEAT` is allowed only while `c < max` and +/// increments it, and `EXIT_LOOP` is allowed only once `c >= min`. The state count is therefore +/// independent of `min` and `max`: `A{1,10000}` compiles to the same automaton shape as `A+`. +/// Registers are per quantifier *instance* and are reset by `START_LOOP`, so nested and repeated +/// quantifiers such as `(A{2}){3}` count independently. +/// +/// ## Empty cycle guard +/// +/// A quantifier whose body can match zero rows, such as `(A*)*` or `(A?)+`, would otherwise let the +/// matcher take the `REPEAT` edge forever without ever consuming a row. `REPEAT` therefore also requires +/// that the previous iteration of the same register consumed at least one row, and `EXIT_LOOP` is permitted +/// unconditionally right after an empty iteration - an unbounded number of empty iterations trivially satisfies any +/// minimum. Termination is guaranteed without rejecting any legal pattern. +/// +/// ## Anchors +/// +/// `^` and `$` become guard transitions that consume no row and are satisfied only at the first row of the +/// partition, and past its last row, respectively. They are relative to the partition, as SQL:2016 requires, not to +/// the input as a whole. +/// +/// This class is stateless; [#compile] allocates its own builder state. +public class PatternToNfaCompiler { + private PatternToNfaCompiler() { + } + + /// Compiles `pattern` into an automaton with prioritized transitions. + /// + /// @throws org.apache.pinot.spi.exception.QueryException if the pattern contains a construct that is not supported + /// yet. Every such construct is rejected here rather than approximated, because an approximated pattern + /// returns wrong rows rather than an error. + public static PatternNfa compile(RowPattern pattern) { + Builder builder = new Builder(); + int start = builder.newState(); + int accept = builder.compileInto(pattern, start); + return new PatternNfa(builder._states, start, accept, builder._numCounters); + } + + /// Mutable automaton under construction. [#compileInto] appends the states of one sub-pattern and returns the + /// state the automaton is in after that sub-pattern matched. + private static class Builder { + private final List _states = new ArrayList<>(); + private int _numCounters; + + private int newState() { + _states.add(new State()); + return _states.size() - 1; + } + + private void addTransition(int from, TransitionKind kind, int target, int operand, int bound) { + _states.get(from).addTransition(new Transition(kind, target, operand, bound)); + } + + private int compileInto(RowPattern pattern, int from) { + switch (pattern.getKind()) { + case SYMBOL: + return compileSymbol((RowPattern.Symbol) pattern, from); + case CONCAT: + return compileConcat((RowPattern.Concat) pattern, from); + case ALTERNATE: + return compileAlternate((RowPattern.Alternate) pattern, from); + case QUANTIFIER: + return compileQuantifier((RowPattern.Quantifier) pattern, from); + case ANCHOR_START: + return compileAnchor(TransitionKind.ANCHOR_START, from); + case ANCHOR_END: + return compileAnchor(TransitionKind.ANCHOR_END, from); + default: + throw QueryErrorCode.QUERY_EXECUTION.asException( + "Unsupported MATCH_RECOGNIZE PATTERN construct: " + pattern.getKind()); + } + } + + private int compileSymbol(RowPattern.Symbol symbol, int from) { + int to = newState(); + addTransition(from, TransitionKind.MATCH, to, symbol.getSymbolOrdinal(), 0); + return to; + } + + private int compileConcat(RowPattern.Concat concat, int from) { + int current = from; + for (RowPattern child : concat.getChildren()) { + current = compileInto(child, current); + } + return current; + } + + /// Emits one epsilon transition per branch, in source order. The transition list order is the preference order, so + /// the leftmost branch that lets the whole pattern complete wins, as SQL:2016 requires. + private int compileAlternate(RowPattern.Alternate alternate, int from) { + int to = newState(); + for (RowPattern child : alternate.getChildren()) { + int branchStart = newState(); + addTransition(from, TransitionKind.EPSILON, branchStart, 0, 0); + int branchEnd = compileInto(child, branchStart); + addTransition(branchEnd, TransitionKind.EPSILON, to, 0, 0); + } + return to; + } + + /// Emits the counter guarded cycle described in the class javadoc. The two loop head transitions are ordered + /// `REPEAT` first for a greedy quantifier and `EXIT_LOOP` first for a reluctant one; that ordering is + /// the only difference between the two. + private int compileQuantifier(RowPattern.Quantifier quantifier, int from) { + int counterId = _numCounters++; + int loopHead = newState(); + int exit = newState(); + int bodyStart = newState(); + addTransition(from, TransitionKind.START_LOOP, loopHead, counterId, 0); + int bodyEnd = compileInto(quantifier.getChild(), bodyStart); + addTransition(bodyEnd, TransitionKind.EPSILON, loopHead, 0, 0); + + int maxRepeat = quantifier.getMaxRepeat() == RowPattern.Quantifier.UNBOUNDED ? PatternNfa.UNBOUNDED + : quantifier.getMaxRepeat(); + if (quantifier.isGreedy()) { + addTransition(loopHead, TransitionKind.REPEAT, bodyStart, counterId, maxRepeat); + addTransition(loopHead, TransitionKind.EXIT_LOOP, exit, counterId, quantifier.getMinRepeat()); + } else { + addTransition(loopHead, TransitionKind.EXIT_LOOP, exit, counterId, quantifier.getMinRepeat()); + addTransition(loopHead, TransitionKind.REPEAT, bodyStart, counterId, maxRepeat); + } + return exit; + } + + private int compileAnchor(TransitionKind kind, int from) { + int to = newState(); + addTransition(from, kind, to, 0, 0); + return to; + } + } +} diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java index 9ebb0d185e1e..c7a07ea91a68 100644 --- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java @@ -35,6 +35,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -50,6 +51,7 @@ import org.apache.pinot.query.runtime.operator.LiteralValueOperator; import org.apache.pinot.query.runtime.operator.MailboxReceiveOperator; import org.apache.pinot.query.runtime.operator.MailboxSendOperator; +import org.apache.pinot.query.runtime.operator.MatchOperator; import org.apache.pinot.query.runtime.operator.MultiStageOperator; import org.apache.pinot.query.runtime.operator.OpChain; import org.apache.pinot.query.runtime.operator.RepeatOperator; @@ -425,6 +427,18 @@ public MultiStageOperator visitExplained(ExplainedNode node, OpChainExecutionCon "Plan node of type ExplainedNode is not supported in OpChain execution."); } + @Override + public MultiStageOperator visitMatch(MatchNode node, OpChainExecutionContext context) { + MultiStageOperator child = null; + try { + PlanNode input = node.getInputs().get(0); + child = visit(input, context); + return new MatchOperator(context, child, input.getDataSchema(), node); + } catch (Exception e) { + return new ErrorOperator(context, QueryErrorCode.QUERY_EXECUTION, e.getMessage(), child); + } + } + @Override public MultiStageOperator visitUnnest(UnnestNode node, OpChainExecutionContext context) { MultiStageOperator child = null; diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java index 2a3c35429bed..92c28ae40449 100644 --- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java +++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java @@ -38,6 +38,7 @@ import org.apache.pinot.query.planner.plannode.JoinNode; import org.apache.pinot.query.planner.plannode.MailboxReceiveNode; import org.apache.pinot.query.planner.plannode.MailboxSendNode; +import org.apache.pinot.query.planner.plannode.MatchNode; import org.apache.pinot.query.planner.plannode.PlanNode; import org.apache.pinot.query.planner.plannode.PlanNodeVisitor; import org.apache.pinot.query.planner.plannode.ProjectNode; @@ -337,6 +338,15 @@ public Void visitUnnest(UnnestNode node, ServerPlanRequestContext context) { return null; } + @Override + public Void visitMatch(MatchNode node, ServerPlanRequestContext context) { + if (visit(node.getInputs().get(0), context)) { + // MATCH_RECOGNIZE is not runnable on leaf, use its input as the boundary. + context.setLeafStageBoundaryNode(node.getInputs().get(0)); + } + return null; + } + private boolean visit(PlanNode node, ServerPlanRequestContext context) { node.visit(this, context); return context.getLeafStageBoundaryNode() == null; diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerMatchLimitsTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerMatchLimitsTest.java new file mode 100644 index 000000000000..1ce99512989f --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerMatchLimitsTest.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime; + +import java.util.HashMap; +import java.util.Map; +import org.apache.pinot.common.utils.config.QueryOptionsUtils; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + + +/// Verifies query-over-server precedence for the MATCH_RECOGNIZE limits copied into op-chain metadata. +public class QueryRunnerMatchLimitsTest { + @Test + public void testCanonicalQueryOptionsTakePrecedenceOverServerDefaults() { + Map opChainMetadata = QueryOptionsUtils.resolveCaseInsensitiveOptions(Map.of( + "MAXROWSINMATCHPARTITION", "7", "MAXSTEPSPERMATCHATTEMPT", "9")); + + QueryRunner.applyMatchLimitDefaults(opChainMetadata, 100, 200L); + + assertEquals(opChainMetadata.get(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION), "7"); + assertEquals(opChainMetadata.get(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT), "9"); + } + + @Test + public void testServerDefaultsAreAddedWhenQueryOptionsAreAbsent() { + Map opChainMetadata = new HashMap<>(); + + QueryRunner.applyMatchLimitDefaults(opChainMetadata, 100, 200L); + + assertEquals(opChainMetadata.get(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION), "100"); + assertEquals(opChainMetadata.get(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT), "200"); + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MatchOperatorTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MatchOperatorTest.java new file mode 100644 index 000000000000..aa3da8f0f403 --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MatchOperatorTest.java @@ -0,0 +1,702 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.planner.plannode.MatchNode; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.planner.plannode.PlanNode; +import org.apache.pinot.query.planner.plannode.RowPattern; +import org.apache.pinot.query.runtime.blocks.ErrorMseBlock; +import org.apache.pinot.query.runtime.blocks.MseBlock; +import org.apache.pinot.query.runtime.operator.match.MatchLimits; +import org.apache.pinot.query.runtime.operator.match.MatchTestFixtures; +import org.apache.pinot.query.runtime.plan.OpChainExecutionContext; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.exception.QueryException; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.INPUT_SCHEMA; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.VALUE_INDEX; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.anySymbol; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.concat; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.labelSymbols; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.quantifier; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.rows; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.symbol; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// End to end tests of [MatchOperator]: measures, the four AFTER MATCH SKIP modes, empty matches, partition +/// handling and both resource guardrails. +public class MatchOperatorTest { + /// Two pattern variables that match every row, so only the pattern shape decides the classification. + private static final List ANY_A_B = List.of(anySymbol("A"), anySymbol("B")); + + @Test + public void testEmitsOneRowPerMatchWithMeasures() { + // PATTERN (A B+) DEFINE A AS label = 'A', B AS label = 'B' over A B B A B. + RowPattern pattern = concat(symbol(0), quantifier(symbol(1), 1, RowPattern.Quantifier.UNBOUNDED, true)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno", "cls", "firstA", "lastB", "sumB", "cnt"}, + new ColumnDataType[]{ + ColumnDataType.INT, ColumnDataType.LONG, ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.INT, + ColumnDataType.LONG, ColumnDataType.LONG + }); + List measures = List.of( + measure("mno", matchNumber()), + measure("cls", classifier()), + measure("firstA", navigation("FIRST", ColumnDataType.INT, patternRef(0, "A"), 0)), + measure("lastB", navigation("LAST", ColumnDataType.INT, patternRef(1, "B"), 0)), + measure("sumB", aggregate("SUM", ColumnDataType.LONG, patternRef(1, "B"))), + measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + List output = run(node(resultSchema, labelSymbols("A", "B"), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("ABBAB")); + + assertEquals(output.size(), 2); + // First match covers rows 0..2: A, B, B. + assertEquals(output.get(0), new Object[]{0, 1L, "B", 0, 2, 3L, 3L}); + // Second match covers rows 3..4: A, B. MATCH_NUMBER restarts nothing, it keeps counting within the partition. + assertEquals(output.get(1), new Object[]{0, 2L, "B", 3, 4, 4L, 2L}); + } + + /// Every single-variable aggregate other than SUM, which is the only one the rest of this class exercises. + /// COUNT over an explicit argument used to throw `Unexpected MATCH_RECOGNIZE aggregate: COUNT` because the + /// accumulation loop ran for COUNT as well, and `accumulate` has no COUNT branch. + @Test + public void testSingleVariableAggregates() { + // PATTERN (A B+) over A B B A B, where the value column equals the row index. + RowPattern pattern = concat(symbol(0), quantifier(symbol(1), 1, RowPattern.Quantifier.UNBOUNDED, true)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "countB", "minB", "maxB", "avgB"}, + new ColumnDataType[]{ + ColumnDataType.INT, ColumnDataType.LONG, ColumnDataType.INT, ColumnDataType.INT, ColumnDataType.DOUBLE + }); + List measures = List.of( + measure("countB", aggregate("COUNT", ColumnDataType.LONG, patternRef(1, "B"))), + measure("minB", aggregate("MIN", ColumnDataType.INT, patternRef(1, "B"))), + measure("maxB", aggregate("MAX", ColumnDataType.INT, patternRef(1, "B"))), + measure("avgB", aggregate("AVG", ColumnDataType.DOUBLE, patternRef(1, "B")))); + + List output = run(node(resultSchema, labelSymbols("A", "B"), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("ABBAB")); + + assertEquals(output.size(), 2); + // First match binds rows 1 and 2 to B, so the aggregated values are 1 and 2. + assertEquals(output.get(0), new Object[]{0, 2L, 1, 2, 1.5}); + // Second match binds only row 4 to B. + assertEquals(output.get(1), new Object[]{0, 1L, 4, 4, 4.0}); + } + + @Test(dataProvider = "skipModes") + public void testAfterMatchSkipModes(MatchNode.AfterMatchSkipMode skipMode, int skipToSymbolOrdinal, + List expectedLastValues) { + // PATTERN (A A B B) with both variables matching every row, over seven rows valued 0..6. Every skip mode resumes + // at a different place, so the set of matches differs. + RowPattern pattern = concat(symbol(0), symbol(0), symbol(1), symbol(1)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastB"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("lastB", navigation("LAST", ColumnDataType.INT, patternRef(1, "B"), 0))); + + List output = + run(node(resultSchema, ANY_A_B, pattern, measures, skipMode, skipToSymbolOrdinal), rows("0123456")); + + assertEquals(lastValues(output), expectedLastValues); + } + + /// The second column of every output row, which the skip mode tests use to identify the matches that were reported. + private static List lastValues(List output) { + List values = new ArrayList<>(output.size()); + for (Object[] row : output) { + values.add((Integer) row[1]); + } + return values; + } + + @DataProvider(name = "skipModes") + public Object[][] skipModes() { + //@formatter:off + return new Object[][]{ + // Resume past row 3, so only rows 4..6 are left and they are too few for another match. + {MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL, List.of(3)}, + // Resume one row after the start of the match every time, so the matches overlap maximally. + {MatchNode.AfterMatchSkipMode.TO_NEXT_ROW, MatchNode.NO_SKIP_TO_SYMBOL, List.of(3, 4, 5, 6)}, + // The first row mapped to B is two rows after the start of the match. + {MatchNode.AfterMatchSkipMode.TO_FIRST, 1, List.of(3, 5)}, + // The last row mapped to B is the last row of the match. + {MatchNode.AfterMatchSkipMode.TO_LAST, 1, List.of(3, 6)} + }; + //@formatter:on + } + + @Test + public void testSkipToLastResumesAtThatVariableRatherThanAtTheEndOfTheMatch() { + // PATTERN (A A B C), all variables matching every row, over seven rows valued 0..6. The last row mapped to B sits + // in the middle of the match, so SKIP TO LAST B must resume there and not at the end of the match. Resuming at the + // end would report the same rows as SKIP PAST LAST ROW, which is the bug this pins down. + RowPattern pattern = concat(symbol(0), symbol(0), symbol(1), symbol(2)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastC"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("lastC", navigation("LAST", ColumnDataType.INT, patternRef(2, "C"), 0))); + List symbols = List.of(anySymbol("A"), anySymbol("B"), anySymbol("C")); + + List toLast = run(node(resultSchema, symbols, pattern, measures, + MatchNode.AfterMatchSkipMode.TO_LAST, 1), rows("0123456")); + // Match at row 0 covers rows 0..3 and resumes at row 2, the row mapped to B; the next match covers rows 2..5. + assertEquals(lastValues(toLast), List.of(3, 5)); + + List pastLastRow = run(node(resultSchema, symbols, pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("0123456")); + // Resuming past row 3 leaves only three rows, which is one too few for another match. + assertEquals(lastValues(pastLastRow), List.of(3)); + } + + @Test + public void testPartitionSplitAcrossInputBlocksIsMatchedAsOnePartition() { + // A match must be found across an input block boundary: blocks are a transport detail, partitions are not. + RowPattern pattern = concat(symbol(0), symbol(0), symbol(0)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + MatchNode node = node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL); + + OpChainExecutionContext context = OperatorTestUtil.getContext(Map.of()); + List partitionRows = rows(0, "AAA"); + // One row in the first block, the remaining two in the second: no single block holds a whole match. + MultiStageOperator input = new BlockListMultiStageOperator.Builder(context, INPUT_SCHEMA) + .addRow(partitionRows.get(0)).finishBlock() + .addRow(partitionRows.get(1)).addRow(partitionRows.get(2)).finishBlock() + .buildWithEos(); + + MatchOperator operator = new MatchOperator(context, input, INPUT_SCHEMA, node); + List output = new ArrayList<>(); + MseBlock block = operator.nextBlock(); + while (!block.isEos()) { + output.addAll(((MseBlock.Data) block).asRowHeap().getRows()); + block = operator.nextBlock(); + } + assertTrue(!((MseBlock.Eos) block).isError(), "Expecting a successful end of stream, got: " + block); + assertEquals(output.size(), 1); + assertEquals(output.get(0), new Object[]{0, 3L}); + } + + @Test + public void testInputErrorTakesPrecedenceOverBufferedOutput() { + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + MatchNode node = node(resultSchema, List.of(anySymbol("A")), symbol(0), + List.of(measure("mno", matchNumber())), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL); + OpChainExecutionContext context = OperatorTestUtil.getContext(Map.of()); + MultiStageOperator input = new BlockListMultiStageOperator.Builder(context, INPUT_SCHEMA) + .addRow(0, "A", 0).addRow(1, "A", 1).finishBlock() + .buildWithError(ErrorMseBlock.fromError(QueryErrorCode.INTERNAL, "upstream failed")); + + MseBlock block = new MatchOperator(context, input, INPUT_SCHEMA, node).nextBlock(); + + assertErrorContains(block, QueryErrorCode.INTERNAL, "upstream failed"); + } + + @Test + public void testEmptyMatchIsEmittedAndAlwaysAdvancesOneRow() { + // PATTERN (A*) over B A B: rows that are not an A still produce an empty match, and the scan must move on + // instead of matching the same empty match forever. + RowPattern pattern = quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG, ColumnDataType.LONG}); + List measures = List.of(measure("mno", matchNumber()), + measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + List output = run(node(resultSchema, labelSymbols("A"), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("BAB")); + + assertEquals(output.size(), 3); + assertEquals(output.get(0), new Object[]{0, 1L, 0L}); + assertEquals(output.get(1), new Object[]{0, 2L, 1L}); + assertEquals(output.get(2), new Object[]{0, 3L, 0L}); + } + + @Test + public void testSkipToFirstThatCannotProgressIsAnError() { + // AFTER MATCH SKIP TO FIRST A on PATTERN (A+) would resume at the first row of the match it just reported, which + // never terminates. SQL:2016 makes that an error rather than a silently adjusted position. + RowPattern pattern = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + MseBlock block = execute(node(resultSchema, labelSymbols("A"), pattern, measures, + MatchNode.AfterMatchSkipMode.TO_FIRST, 0), rows("AAA"), Map.of()); + + assertErrorContains(block, QueryErrorCode.QUERY_EXECUTION, "would resume at the first row of the match"); + } + + @Test + public void testMatchesAreScopedToTheirPartition() { + // PATTERN (A A) over two partitions of two rows each: a match must never span the boundary. + RowPattern pattern = concat(symbol(0), symbol(0)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastA"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("lastA", navigation("LAST", ColumnDataType.INT, patternRef(0, "A"), 0))); + + List partitioned = new ArrayList<>(rows(0, "AA")); + partitioned.addAll(rows(1, "AA")); + List output = run(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), partitioned); + + assertEquals(output.size(), 2); + assertEquals(output.get(0), new Object[]{0, 1}); + assertEquals(output.get(1), new Object[]{1, 1}); + } + + @Test + public void testUngroupedPartitionsAreRejected() { + // The sort exchange below the operator guarantees that a partition arrives in one contiguous run. If it does not, + // matching each fragment separately would report matches that do not exist, so fail instead. + RowPattern pattern = concat(symbol(0), symbol(0)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastA"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("lastA", navigation("LAST", ColumnDataType.INT, patternRef(0, "A"), 0))); + + List interleaved = new ArrayList<>(rows(0, "AA")); + interleaved.addAll(rows(1, "AA")); + interleaved.addAll(rows(0, "AA")); + MseBlock block = execute(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), interleaved, Map.of()); + + assertErrorContains(block, QueryErrorCode.QUERY_EXECUTION, "is not grouped by partition key"); + } + + @Test + public void testOnePartitionEmitsBoundedOutputBlocks() { + int numRows = 2 * MatchOperator.MAX_OUTPUT_ROWS_PER_BLOCK + 17; + List inputRows = new ArrayList<>(numRows); + for (int i = 0; i < numRows; i++) { + inputRows.add(new Object[]{0, "A", i}); + } + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + MatchNode node = node(resultSchema, List.of(anySymbol("A")), symbol(0), + List.of(measure("mno", matchNumber())), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL); + + OpChainExecutionContext context = OperatorTestUtil.getContext(Map.of()); + BlockListMultiStageOperator.Builder builder = new BlockListMultiStageOperator.Builder(context, INPUT_SCHEMA); + for (Object[] row : inputRows) { + builder.addRow(row); + } + MatchOperator operator = new MatchOperator(context, builder.finishBlock().buildWithEos(), INPUT_SCHEMA, node); + int outputRows = 0; + int outputBlocks = 0; + long expectedMatchNumber = 1; + MseBlock block = operator.nextBlock(); + while (!block.isEos()) { + MseBlock.Data dataBlock = (MseBlock.Data) block; + int blockRows = dataBlock.getNumRows(); + assertTrue(blockRows > 0 && blockRows <= MatchOperator.MAX_OUTPUT_ROWS_PER_BLOCK); + for (Object[] row : dataBlock.asRowHeap().getRows()) { + assertEquals(row, new Object[]{0, expectedMatchNumber++}); + } + outputRows += blockRows; + outputBlocks++; + block = operator.nextBlock(); + } + + assertTrue(!((MseBlock.Eos) block).isError(), "Expecting a successful end of stream, got: " + block); + assertEquals(outputRows, numRows); + assertEquals(outputBlocks, 3); + assertEquals(expectedMatchNumber, numRows + 1L); + } + + @Test + public void testHighCardinalityPartitionsUseConstantValidationState() { + int numPartitions = 10_000; + List inputRows = new ArrayList<>(numPartitions); + for (int partition = 0; partition < numPartitions; partition++) { + inputRows.add(new Object[]{partition, "A", partition}); + } + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List output = run(node(resultSchema, List.of(anySymbol("A")), symbol(0), + List.of(measure("mno", matchNumber())), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL), inputRows); + + assertEquals(output.size(), numPartitions); + assertEquals(output.get(0), new Object[]{0, 1L}); + assertEquals(output.get(numPartitions - 1), new Object[]{numPartitions - 1, 1L}); + } + + @Test + public void testMultiValuePartitionKeyIsRejectedAtRuntime() { + DataSchema inputSchema = new DataSchema(new String[]{"pid", "label", "value"}, + new ColumnDataType[]{ColumnDataType.INT_ARRAY, ColumnDataType.STRING, ColumnDataType.INT}); + DataSchema resultSchema = new DataSchema(new String[]{"pid"}, + new ColumnDataType[]{ColumnDataType.INT_ARRAY}); + MatchNode node = node(resultSchema, List.of(anySymbol("A")), symbol(0), List.of(), + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL); + MultiStageOperator input = new BlockListMultiStageOperator.Builder(inputSchema).addBlock().buildWithEos(); + + QueryException exception = expectThrows(QueryException.class, + () -> new MatchOperator(OperatorTestUtil.getTracingContext(), input, inputSchema, node)); + assertTrue(exception.getMessage().contains("requires single-value columns"), exception.getMessage()); + } + + @Test + public void testMaxRowsInMatchPartitionThrowsRatherThanTruncating() { + RowPattern pattern = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + MseBlock block = execute(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA"), + Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, "2")); + + assertErrorContains(block, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED, + "exceeds the maximum of 2 rows"); + } + + @Test + public void testMaxStepsPerMatchAttemptThrowsRatherThanTruncating() { + RowPattern pattern = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + MseBlock block = execute(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAAAAAAAA"), + Map.of(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, "3")); + + assertErrorContains(block, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED, + "maximum of 3 pattern matching steps"); + } + + @Test + public void testHintOutranksTheQueryOption() { + RowPattern pattern = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + PlanNode.NodeHint hint = new PlanNode.NodeHint( + Map.of(MatchLimits.MATCH_HINT_OPTIONS, Map.of(MatchLimits.MAX_ROWS_IN_MATCH_PARTITION_HINT, "2"))); + + MatchNode node = new MatchNode(-1, resultSchema, hint, List.of(), List.of(anySymbol("A")), pattern, measures, + List.of(MatchTestFixtures.PID_INDEX), List.of(), MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, + MatchNode.NO_SKIP_TO_SYMBOL, MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + // The query option is far more permissive, but the hint wins. + MseBlock block = execute(node, rows("AAAA"), Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, "1000")); + + assertErrorContains(block, QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED, "exceeds the maximum of 2 rows"); + } + + @Test + public void testAllRowsPerMatchIsRejected() { + RowPattern pattern = symbol(0); + DataSchema resultSchema = new DataSchema(new String[]{"pid"}, new ColumnDataType[]{ColumnDataType.INT}); + MatchNode node = new MatchNode(-1, resultSchema, PlanNode.NodeHint.EMPTY, List.of(), List.of(anySymbol("A")), + pattern, List.of(), List.of(MatchTestFixtures.PID_INDEX), List.of(), + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL, + MatchNode.RowsPerMatchMode.ALL_ROWS_PER_MATCH); + MultiStageOperator input = new BlockListMultiStageOperator.Builder(INPUT_SCHEMA).addBlock().buildWithEos(); + + QueryException exception = expectThrows(QueryException.class, + () -> new MatchOperator(OperatorTestUtil.getTracingContext(), input, INPUT_SCHEMA, node)); + assertTrue(exception.getMessage().contains("ALL ROWS PER MATCH"), exception.getMessage()); + } + + @Test + public void testDefinePredicateCanNavigateToEarlierRows() { + // DEFINE B AS value > PREV(value, 1): an increasing run. Values are the row indexes, so every row qualifies once + // the first one anchored the run. + RexExpression current = new RexExpression.FunctionCall(ColumnDataType.INT, "PREV", + List.of(patternRef(0, "A"), literal(0))); + RexExpression previous = new RexExpression.FunctionCall(ColumnDataType.INT, "PREV", + List.of(patternRef(0, "A"), literal(1))); + PatternSymbol increasing = new PatternSymbol("A", + new RexExpression.FunctionCall(ColumnDataType.BOOLEAN, "GREATER_THAN", List.of(current, previous))); + + RowPattern pattern = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + List output = run(node(resultSchema, List.of(increasing), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA")); + + // Row 0 has no previous row, so PREV is null and the predicate is false: the first match starts at row 1 and runs + // to the end of the partition. + assertEquals(output.size(), 1); + assertEquals(output.get(0), new Object[]{0, 3L}); + } + + @Test + public void testDefinePredicateCanReferenceAnotherPatternVariable() { + // DEFINE A AS label = 'A', B AS B.value > A.value. The cross symbol reference must resolve to the last row mapped + // to A so far, not to the row being tested. Row 2 breaks the run precisely because A.value is still 5, so an + // implementation that resolved A.value to the current row would compare a value with itself and match nothing. + PatternSymbol a = MatchTestFixtures.labelSymbol("A", 0); + RexExpression bValue = new RexExpression.FunctionCall(ColumnDataType.INT, "PREV", + List.of(patternRef(1, "B"), literal(0))); + RexExpression aValue = new RexExpression.FunctionCall(ColumnDataType.INT, "PREV", + List.of(patternRef(0, "A"), literal(0))); + PatternSymbol b = new PatternSymbol("B", + new RexExpression.FunctionCall(ColumnDataType.BOOLEAN, "GREATER_THAN", List.of(bValue, aValue))); + + RowPattern pattern = concat(symbol(0), quantifier(symbol(1), 1, RowPattern.Quantifier.UNBOUNDED, true)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastB", "cnt"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT, ColumnDataType.LONG}); + List measures = List.of( + measure("lastB", navigation("LAST", ColumnDataType.INT, patternRef(1, "B"), 0)), + measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + + List inputRows = List.of(new Object[]{0, "A", 5}, new Object[]{0, "B", 6}, new Object[]{0, "B", 3}, + new Object[]{0, "B", 7}); + List output = run(node(resultSchema, List.of(a, b), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), inputRows); + + // The run stops at value 3, which is not greater than 5, so the match is rows 0..1 only. Rows 2 and 3 cannot start + // a match of their own because neither is labelled 'A'. + assertEquals(output.size(), 1); + assertEquals(output.get(0), new Object[]{0, 6, 2L}); + } + + @Test + public void testNavigationMayLeaveTheMatchButNotThePartition() { + // NEXT(A.value, 1) reads the row after the last row of the match. That row is outside the match but inside the + // partition, which SQL:2016 allows: a physical step is bounded by the partition only. Stepping past the end of the + // partition yields null rather than the first row of the next partition. + RowPattern pattern = concat(symbol(0), symbol(0)); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "next"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("next", navigation("NEXT", ColumnDataType.INT, patternRef(0, "A"), 1))); + + List partitioned = new ArrayList<>(rows(0, "AAAA")); + partitioned.addAll(rows(1, "AA")); + List output = run(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), partitioned); + + assertEquals(output.size(), 3); + // Partition 0: the match on rows 0..1 sees row 2, the one on rows 2..3 falls off the end of the partition. + assertEquals(output.get(0), new Object[]{0, 2}); + assertEquals(output.get(1), new Object[]{0, null}); + // Partition 1: the only match covers the whole partition, so there is no next row here either. + assertEquals(output.get(2), new Object[]{1, null}); + } + + @Test + public void testMatchNumberRestartsInEachPartition() { + // SQL:2016 numbers matches within a partition, so the counter must restart at the partition boundary rather than + // run on across the whole input. + DataSchema resultSchema = new DataSchema(new String[]{"pid", "mno"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.LONG}); + List measures = List.of(measure("mno", matchNumber())); + + List partitioned = new ArrayList<>(rows(0, "AA")); + partitioned.addAll(rows(1, "AA")); + List output = run(node(resultSchema, List.of(anySymbol("A")), symbol(0), measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), partitioned); + + assertEquals(output.size(), 4); + assertEquals(output.get(0), new Object[]{0, 1L}); + assertEquals(output.get(1), new Object[]{0, 2L}); + assertEquals(output.get(2), new Object[]{1, 1L}); + assertEquals(output.get(3), new Object[]{1, 2L}); + } + + @Test + public void testLogicalOffsetSelectsTheNthRowMappedToTheVariable() { + // FIRST(A.value, 1) is the second row mapped to A and LAST(A.value, 1) is the second to last, so over a match of + // four rows valued 0..3 they must be different rows. An offset that was ignored would make them equal. + RowPattern pattern = quantifier(symbol(0), 4, 4, true); + DataSchema resultSchema = new DataSchema(new String[]{"pid", "first1", "last1"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT, ColumnDataType.INT}); + List measures = List.of( + measure("first1", navigation("FIRST", ColumnDataType.INT, patternRef(0, "A"), 1)), + measure("last1", navigation("LAST", ColumnDataType.INT, patternRef(0, "A"), 1))); + + List output = run(node(resultSchema, List.of(anySymbol("A")), pattern, measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA")); + + assertEquals(output.size(), 1); + assertEquals(output.get(0), new Object[]{0, 1, 2}); + } + + @Test + public void testAnchorsRestrictWhereAMatchMayStartAndEnd() { + // PATTERN (^ A A) over four rows: without the anchor the scan would report a second match on rows 2..3. + DataSchema resultSchema = new DataSchema(new String[]{"pid", "lastA"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.INT}); + List measures = + List.of(measure("lastA", navigation("LAST", ColumnDataType.INT, patternRef(0, "A"), 0))); + List symbols = List.of(anySymbol("A")); + + List anchoredStart = run(node(resultSchema, symbols, + concat(RowPattern.AnchorStart.INSTANCE, symbol(0), symbol(0)), measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA")); + assertEquals(lastValues(anchoredStart), List.of(1)); + + // PATTERN (A A $) only matches the pair that ends at the last row of the partition. + List anchoredEnd = run(node(resultSchema, symbols, + concat(symbol(0), symbol(0), RowPattern.AnchorEnd.INSTANCE), measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA")); + assertEquals(lastValues(anchoredEnd), List.of(3)); + + // Unanchored, the same pattern reports both pairs, which is what the anchors are being compared against. + List unanchored = run(node(resultSchema, symbols, concat(symbol(0), symbol(0)), measures, + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL), rows("AAAA")); + assertEquals(lastValues(unanchored), List.of(1, 3)); + } + + @Test + public void testPartitionKeysAreEmittedInTheOrderTheSchemaDeclaresThem() { + // PARTITION BY label, pid: the partition key list is not in ascending input-column order, so the operator must + // fill output slot i from _partitionKeys[i] rather than from the i-th smallest input index. Reading the keys off + // Calcite's ImmutableBitSet instead would put pid's Integer in the STRING slot and blow up in TypeUtils. + DataSchema resultSchema = new DataSchema(new String[]{"label", "pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + MatchNode node = new MatchNode(-1, resultSchema, PlanNode.NodeHint.EMPTY, List.of(), List.of(anySymbol("A")), + concat(symbol(0), symbol(0)), + measures, List.of(MatchTestFixtures.LABEL_INDEX, MatchTestFixtures.PID_INDEX), List.of(), + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + + List output = run(node, rows("AA")); + + assertEquals(output.size(), 1); + assertEquals(output.get(0), new Object[]{"A", 0, 2L}); + } + + @Test + public void testPartitionGroupingIsValidatedInExchangeCollationOrder() { + // PARTITION BY label, pid is emitted in SQL order, but the exchange sorts the input fields by ordinal: pid, label. + // These rows increase in exchange order and decrease in SQL order ("Z" then "A"), so using the output-key order + // for monotonic validation would reject valid input. + DataSchema resultSchema = new DataSchema(new String[]{"label", "pid", "cnt"}, + new ColumnDataType[]{ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.LONG}); + List measures = + List.of(measure("cnt", new RexExpression.FunctionCall(ColumnDataType.LONG, "COUNT", List.of()))); + MatchNode node = new MatchNode(-1, resultSchema, PlanNode.NodeHint.EMPTY, List.of(), List.of(anySymbol("A")), + symbol(0), measures, List.of(MatchTestFixtures.LABEL_INDEX, MatchTestFixtures.PID_INDEX), List.of(), + MatchNode.AfterMatchSkipMode.PAST_LAST_ROW, MatchNode.NO_SKIP_TO_SYMBOL, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + + List output = run(node, List.of(new Object[]{0, "Z", 0}, new Object[]{1, "A", 0})); + + assertEquals(output.size(), 2); + assertEquals(output.get(0), new Object[]{"Z", 0, 1L}); + assertEquals(output.get(1), new Object[]{"A", 1, 1L}); + } + + private static MatchNode node(DataSchema resultSchema, List symbols, RowPattern pattern, + List measures, MatchNode.AfterMatchSkipMode skipMode, int skipToSymbolOrdinal) { + return new MatchNode(-1, resultSchema, PlanNode.NodeHint.EMPTY, List.of(), symbols, pattern, measures, + List.of(MatchTestFixtures.PID_INDEX), List.of(), skipMode, skipToSymbolOrdinal, + MatchNode.RowsPerMatchMode.ONE_ROW_PER_MATCH); + } + + /// Drains the operator and returns every output row. The operator emits a block per partition, so a test that spans + /// partitions must not stop at the first block. + private static List run(MatchNode node, List inputRows) { + List outputRows = new ArrayList<>(); + MseBlock.Eos terminal = drain(node, inputRows, Map.of(), outputRows); + assertTrue(!terminal.isError(), "Expecting a successful end of stream, got: " + terminal); + return outputRows; + } + + private static MseBlock execute(MatchNode node, List inputRows, Map opChainMetadata) { + return drain(node, inputRows, opChainMetadata, new ArrayList<>()); + } + + private static MseBlock.Eos drain(MatchNode node, List inputRows, Map opChainMetadata, + List outputRows) { + OpChainExecutionContext context = OperatorTestUtil.getContext(opChainMetadata); + BlockListMultiStageOperator.Builder builder = new BlockListMultiStageOperator.Builder(context, INPUT_SCHEMA); + for (Object[] row : inputRows) { + builder.addRow(row); + } + MultiStageOperator input = (inputRows.isEmpty() ? builder : builder.finishBlock()).buildWithEos(); + MatchOperator operator = new MatchOperator(context, input, INPUT_SCHEMA, node); + while (true) { + MseBlock block = operator.nextBlock(); + if (block.isEos()) { + return (MseBlock.Eos) block; + } + outputRows.addAll(((MseBlock.Data) block).asRowHeap().getRows()); + } + } + + private static void assertErrorContains(MseBlock block, QueryErrorCode errorCode, String fragment) { + assertTrue(block.isError(), "Expecting an error block, got: " + block); + String message = ((ErrorMseBlock) block).getErrorMessages().get(errorCode); + assertTrue(message != null && message.contains(fragment), + "Expecting " + errorCode + " containing '" + fragment + "', got: " + + ((ErrorMseBlock) block).getErrorMessages()); + } + + private static MatchNode.Measure measure(String name, RexExpression expression) { + return new MatchNode.Measure(name, expression); + } + + private static RexExpression patternRef(int symbolOrdinal, String alpha) { + return new RexExpression.PatternFieldRef(VALUE_INDEX, symbolOrdinal, alpha); + } + + private static RexExpression literal(int value) { + return new RexExpression.Literal(ColumnDataType.INT, value); + } + + private static RexExpression navigation(String functionName, ColumnDataType dataType, RexExpression operand, + int offset) { + return new RexExpression.FunctionCall(dataType, functionName, List.of(operand, literal(offset))); + } + + private static RexExpression aggregate(String functionName, ColumnDataType dataType, RexExpression operand) { + return new RexExpression.FunctionCall(dataType, functionName, List.of(operand)); + } + + private static RexExpression matchNumber() { + return new RexExpression.FunctionCall(ColumnDataType.LONG, "MATCH_NUMBER", List.of()); + } + + private static RexExpression classifier() { + return new RexExpression.FunctionCall(ColumnDataType.STRING, "CLASSIFIER", List.of()); + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchExpressionTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchExpressionTest.java new file mode 100644 index 000000000000..05021434508f --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchExpressionTest.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import java.math.BigDecimal; +import java.util.List; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.runtime.operator.operands.ReferenceOperand; +import org.apache.pinot.spi.exception.QueryException; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Focused correctness and allocation regression tests for match-specific expression terms. +public class MatchExpressionTest { + private static final int UNIVERSAL_SYMBOL_ORDINAL = RexExpression.PatternFieldRef.UNIVERSAL_SYMBOL_ORDINAL; + private static final List SYMBOLS = List.of(new PatternSymbol("A", null)); + + @Test + public void testNavigationOffsetMustBeAnExactNonNegativeIntegerInRange() { + assertNavigationCompileFails(new RexExpression.Literal(ColumnDataType.DOUBLE, 1.5), "exact integer"); + assertNavigationCompileFails(new RexExpression.Literal(ColumnDataType.LONG, 1L << 32), + "between 0 and " + Integer.MAX_VALUE); + assertNavigationCompileFails(new RexExpression.Literal(ColumnDataType.INT, -1), "must not be negative"); + } + + @Test + public void testNestedNavigationOffsetOverflowIsRejected() { + RexExpression inner = navigation("NEXT", patternRef(), + new RexExpression.Literal(ColumnDataType.INT, Integer.MAX_VALUE), ColumnDataType.INT); + RexExpression outer = navigation("NEXT", inner, new RexExpression.Literal(ColumnDataType.INT, 1), + ColumnDataType.INT); + + QueryException exception = expectThrows(QueryException.class, + () -> MatchExpression.compile(outer, schema(ColumnDataType.INT))); + + assertTrue(exception.getMessage().contains("combined PREV / NEXT offset"), exception.getMessage()); + } + + @Test + public void testNavigationDoesNotReboxValuesAlreadyInTheDeclaredStoredType() { + Long value = Long.valueOf(1_000); + MatchExpression expression = MatchExpression.compile(patternRef(), schema(ColumnDataType.LONG)); + + Object result = expression.evaluate(tape(List.of(new Object[]{value}))); + + assertSame(result, value); + } + + @Test + public void testNavigationStillConvertsWhenDeclaredStoredTypeDiffers() { + MatchExpression expression = MatchExpression.compile( + navigation("PREV", patternRef(), new RexExpression.Literal(ColumnDataType.INT, 0), ColumnDataType.LONG), + schema(ColumnDataType.INT)); + + Object result = expression.evaluate(tape(List.of(new Object[]{1_000}))); + + assertTrue(result instanceof Long, "Expected a LONG stored value, got: " + result.getClass()); + assertEquals(result, 1_000L); + } + + @Test + public void testCountStarUsesTapeLengthWithoutMaterializingUniversalRows() { + MatchTape tape = new MatchTape(SYMBOLS) { + @Override + public IntArrayList rowsOf(int symbolOrdinal) { + throw new AssertionError("COUNT(*) must not materialize universal row indexes"); + } + }; + tape.reset(List.of(new Object[]{1}, new Object[]{2}), 0, 1); + tape.push(0); + tape.push(0); + MatchTerm.Aggregate count = + new MatchTerm.Aggregate(MatchTerm.Aggregate.Kind.COUNT, UNIVERSAL_SYMBOL_ORDINAL, null, ColumnDataType.LONG); + + assertEquals(count.evaluate(tape), 2L); + } + + @Test + public void testIntegralAndFloatingPointAggregates() { + MatchTerm.Aggregate longSum = aggregate(MatchTerm.Aggregate.Kind.SUM, ColumnDataType.LONG, ColumnDataType.LONG); + List longRows = List.of(new Object[]{9_007_199_254_740_993L}, new Object[]{2L}, new Object[]{null}); + assertEquals(longSum.evaluate(tape(longRows)), 9_007_199_254_740_995L); + + MatchTerm.Aggregate doubleAverage = + aggregate(MatchTerm.Aggregate.Kind.AVG, ColumnDataType.DOUBLE, ColumnDataType.DOUBLE); + List doubleRows = List.of(new Object[]{1.25}, new Object[]{2.75}); + assertEquals(doubleAverage.evaluate(tape(doubleRows)), 2.0); + } + + @Test + public void testBigDecimalSumAndAverageDoNotRoundThroughDouble() { + BigDecimal first = new BigDecimal("9007199254740993.0000000000000001"); + BigDecimal second = new BigDecimal("9007199254740995.0000000000000003"); + List rows = List.of(new Object[]{first}, new Object[]{second}); + + MatchTerm.Aggregate sum = + aggregate(MatchTerm.Aggregate.Kind.SUM, ColumnDataType.BIG_DECIMAL, ColumnDataType.BIG_DECIMAL); + MatchTerm.Aggregate average = + aggregate(MatchTerm.Aggregate.Kind.AVG, ColumnDataType.BIG_DECIMAL, ColumnDataType.BIG_DECIMAL); + + assertEquals(((BigDecimal) sum.evaluate(tape(rows))).compareTo( + new BigDecimal("18014398509481988.0000000000000004")), 0); + assertEquals(((BigDecimal) average.evaluate(tape(rows))).compareTo( + new BigDecimal("9007199254740994.0000000000000002")), 0); + } + + @Test + public void testBigDecimalAggregatesWidenIntegralInputsExactly() { + List rows = List.of(new Object[]{9_007_199_254_740_993L}, new Object[]{1L}); + MatchTerm.Aggregate sum = + aggregate(MatchTerm.Aggregate.Kind.SUM, ColumnDataType.LONG, ColumnDataType.BIG_DECIMAL); + MatchTerm.Aggregate average = + aggregate(MatchTerm.Aggregate.Kind.AVG, ColumnDataType.LONG, ColumnDataType.BIG_DECIMAL); + + assertEquals(sum.evaluate(tape(rows)), new BigDecimal("9007199254740994")); + assertEquals(average.evaluate(tape(rows)), new BigDecimal("4503599627370497")); + } + + @Test + public void testMultiValueAggregateOperandFailsWithActionableError() { + ReferenceOperand argument = new ReferenceOperand(0, schema(ColumnDataType.LONG_ARRAY)); + + QueryException exception = expectThrows(QueryException.class, + () -> new MatchTerm.Aggregate(MatchTerm.Aggregate.Kind.SUM, UNIVERSAL_SYMBOL_ORDINAL, argument, + ColumnDataType.LONG)); + + assertTrue(exception.getMessage().contains("Multi-value operand type 'LONG_ARRAY'"), exception.getMessage()); + assertTrue(exception.getMessage().contains("Reduce the array to a scalar"), exception.getMessage()); + } + + private static void assertNavigationCompileFails(RexExpression.Literal offset, String expectedMessage) { + RexExpression expression = navigation("PREV", patternRef(), offset, ColumnDataType.INT); + QueryException exception = expectThrows(QueryException.class, + () -> MatchExpression.compile(expression, schema(ColumnDataType.INT))); + assertTrue(exception.getMessage().contains(expectedMessage), exception.getMessage()); + } + + private static MatchTerm.Aggregate aggregate(MatchTerm.Aggregate.Kind kind, ColumnDataType inputType, + ColumnDataType resultType) { + return new MatchTerm.Aggregate(kind, UNIVERSAL_SYMBOL_ORDINAL, new ReferenceOperand(0, schema(inputType)), + resultType); + } + + private static MatchTape tape(List rows) { + MatchTape tape = new MatchTape(SYMBOLS); + tape.reset(rows, 0, 1); + for (int i = 0; i < rows.size(); i++) { + tape.push(0); + } + return tape; + } + + private static DataSchema schema(ColumnDataType type) { + return new DataSchema(new String[]{"value"}, new ColumnDataType[]{type}); + } + + private static RexExpression patternRef() { + return new RexExpression.PatternFieldRef(0, 0, "A"); + } + + private static RexExpression navigation(String functionName, RexExpression operand, RexExpression offset, + ColumnDataType resultType) { + return new RexExpression.FunctionCall(resultType, functionName, List.of(operand, offset)); + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchLimitsTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchLimitsTest.java new file mode 100644 index 000000000000..4e76d526bea5 --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchLimitsTest.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.Map; +import org.apache.pinot.query.planner.plannode.PlanNode; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Tests the precedence between the three tiers that can set a MATCH_RECOGNIZE limit, and that a nonsensical value is +/// rejected rather than silently clamped. +/// +/// The cluster config tier is not visible here on purpose: `QueryRunner` folds it into the op chain metadata +/// under the query option key, and only when the query did not set that option itself, so from this class's point of +/// view it is indistinguishable from a query option. That is exactly how `maxRowsInWindow` behaves. +public class MatchLimitsTest { + private static final PlanNode.NodeHint NO_HINT = PlanNode.NodeHint.EMPTY; + + @Test + public void testDefaultsApplyWhenNothingIsSet() { + assertEquals(MatchLimits.getMaxRowsInMatchPartition(Map.of(), NO_HINT), + MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION); + assertEquals(MatchLimits.getMaxStepsPerMatchAttempt(Map.of(), NO_HINT), + MatchLimits.DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT); + // The defaults the task pins: a million rows, and sixteen million steps. + assertEquals(MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION, 1_000_000); + assertEquals(MatchLimits.DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT, 16_000_000L); + } + + @Test + public void testTheStepDefaultDominatesThePerRowCostOfALinearMatch() { + // The two defaults have to be mutually consistent: a linear, zero-backtracking match costs a small constant + // number of transitions per row (3 for PATTERN (A+), 5 for PATTERN ((A|B)+)), so a step budget below that + // constant times the row budget would reject a partition that the row budget explicitly admits. + assertTrue( + MatchLimits.DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT >= 6L * MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION, + "The step default (" + MatchLimits.DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT + ") must dominate the per-row cost " + + "of a linear match over the largest permitted partition (" + + MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION + + " rows)"); + } + + @Test + public void testQueryOptionOverridesTheDefault() { + assertEquals(MatchLimits.getMaxRowsInMatchPartition( + Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, "42"), NO_HINT), 42); + assertEquals( + MatchLimits.getMaxStepsPerMatchAttempt(Map.of(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, "77"), NO_HINT), + 77L); + } + + @Test + public void testHintOverridesTheQueryOption() { + PlanNode.NodeHint hint = hint(Map.of(MatchLimits.MAX_ROWS_IN_MATCH_PARTITION_HINT, "7", + MatchLimits.MAX_STEPS_PER_MATCH_ATTEMPT_HINT, "9")); + Map queryOptions = Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, "1000", + QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, "1000"); + assertEquals(MatchLimits.getMaxRowsInMatchPartition(queryOptions, hint), 7); + assertEquals(MatchLimits.getMaxStepsPerMatchAttempt(queryOptions, hint), 9L); + } + + @Test + public void testHintOfADifferentLimitDoesNotShadowTheQueryOption() { + // A hint that only pins the row limit must leave the step limit resolving through the query option. + PlanNode.NodeHint hint = hint(Map.of(MatchLimits.MAX_ROWS_IN_MATCH_PARTITION_HINT, "7")); + Map queryOptions = Map.of(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, "1000"); + assertEquals(MatchLimits.getMaxRowsInMatchPartition(queryOptions, hint), 7); + assertEquals(MatchLimits.getMaxStepsPerMatchAttempt(queryOptions, hint), 1000L); + } + + @Test + public void testHintsOfOtherOperatorsAreIgnored() { + PlanNode.NodeHint windowHint = new PlanNode.NodeHint(Map.of("windowOptions", Map.of("max_rows_in_window", "3"))); + assertEquals(MatchLimits.getMaxRowsInMatchPartition(Map.of(), windowHint), + MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION); + } + + @Test + public void testNonPositiveAndUnparseableValuesAreRejected() { + // Silently falling back to the default would hide a typo behind a limit the user did not ask for. + for (String bad : new String[]{"0", "-1", "abc", ""}) { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> MatchLimits.getMaxRowsInMatchPartition( + Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, bad), NO_HINT)); + assertTrue(exception.getMessage().contains(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION), exception.getMessage()); + } + } + + @Test + public void testMaxRowsInMatchDoesNotOverflowAnInt() { + // The value is used as an int row count; a long that does not fit must be an error rather than a wrapped value. + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> MatchLimits + .getMaxRowsInMatchPartition( + Map.of(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION, Long.toString(Integer.MAX_VALUE + 1L)), NO_HINT)); + assertTrue(exception.getMessage().contains(QueryOptionKey.MAX_ROWS_IN_MATCH_PARTITION), exception.getMessage()); + // The step budget is a long, so the same value is perfectly legal there. + assertEquals(MatchLimits.getMaxStepsPerMatchAttempt( + Map.of(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT, Long.toString(Integer.MAX_VALUE + 1L)), NO_HINT), + Integer.MAX_VALUE + 1L); + } + + private static PlanNode.NodeHint hint(Map matchOptions) { + return new PlanNode.NodeHint(Map.of(MatchLimits.MATCH_HINT_OPTIONS, matchOptions)); + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchTestFixtures.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchTestFixtures.java new file mode 100644 index 000000000000..d3f431c0b007 --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/MatchTestFixtures.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.common.utils.DataSchema.ColumnDataType; +import org.apache.pinot.query.planner.logical.RexExpression; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.planner.plannode.RowPattern; + + +/// Shared fixtures for the MATCH_RECOGNIZE runtime tests. +/// +/// Rows use a deliberately trivial data model: column `label` carries the name of the pattern variable the +/// row is supposed to match, so a test can describe its input as the string `"AABB"` and read the expected +/// classification straight off it. Column `value` carries an increasing number for tests that need arithmetic or +/// navigation, and `pid` is the partition key. +public final class MatchTestFixtures { + private MatchTestFixtures() { + } + + public static final int PID_INDEX = 0; + public static final int LABEL_INDEX = 1; + public static final int VALUE_INDEX = 2; + + public static final DataSchema INPUT_SCHEMA = new DataSchema(new String[]{"pid", "label", "value"}, + new ColumnDataType[]{ColumnDataType.INT, ColumnDataType.STRING, ColumnDataType.INT}); + + /// A pattern variable whose DEFINE is `label = `, i.e. it matches exactly the rows whose `label` + /// column holds its own name. + public static PatternSymbol labelSymbol(String name, int ordinal) { + RexExpression reference = new RexExpression.PatternFieldRef(LABEL_INDEX, ordinal, name); + RexExpression literal = new RexExpression.Literal(ColumnDataType.STRING, name); + return new PatternSymbol(name, + new RexExpression.FunctionCall(ColumnDataType.BOOLEAN, "EQUALS", List.of(reference, literal))); + } + + /// A pattern variable with no DEFINE clause, which matches every row per SQL:2016. + public static PatternSymbol anySymbol(String name) { + return new PatternSymbol(name, null); + } + + /// Symbol table where each name is a [#labelSymbol], in the order given. + public static List labelSymbols(String... names) { + List symbols = new ArrayList<>(names.length); + for (int i = 0; i < names.length; i++) { + symbols.add(labelSymbol(names[i], i)); + } + return symbols; + } + + public static RowPattern symbol(int ordinal) { + return new RowPattern.Symbol(ordinal); + } + + public static RowPattern concat(RowPattern... children) { + return new RowPattern.Concat(Arrays.asList(children)); + } + + public static RowPattern alternate(RowPattern... children) { + return new RowPattern.Alternate(Arrays.asList(children)); + } + + public static RowPattern quantifier(RowPattern child, int minRepeat, int maxRepeat, boolean greedy) { + return new RowPattern.Quantifier(child, minRepeat, maxRepeat, greedy); + } + + /// Rows of a single partition, one per character of `labels`. The `value` column is the row index, so + /// navigation tests can assert on a value that identifies the row. + public static List rows(String labels) { + return rows(0, labels); + } + + public static List rows(int partitionId, String labels) { + List rows = new ArrayList<>(labels.length()); + for (int i = 0; i < labels.length(); i++) { + rows.add(new Object[]{partitionId, String.valueOf(labels.charAt(i)), i}); + } + return rows; + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcherStepBudgetTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcherStepBudgetTest.java new file mode 100644 index 000000000000..59a6ec60ac90 --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PartitionMatcherStepBudgetTest.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.planner.plannode.RowPattern; +import org.apache.pinot.spi.exception.QueryErrorCode; +import org.apache.pinot.spi.exception.QueryException; +import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey; +import org.testng.annotations.Test; + +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.INPUT_SCHEMA; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.anySymbol; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.quantifier; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.symbol; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Tests `maxStepsPerMatchAttempt` against the linear, zero-backtracking cost of a match rather than against the +/// backtracking it is meant to bound. +/// +/// The step counter counts every automaton transition, including the ones that advance the scan position, so +/// `PATTERN (A+)` with `A` matching every row costs exactly `3n + 4` steps over `n` rows: one +/// REPEAT, one MATCH and one EPSILON per iteration. At the old default of one million steps that capped a linear match +/// at 333,332 rows while [MatchLimits#DEFAULT_MAX_ROWS_IN_MATCH_PARTITION] explicitly admitted three times that, +/// and the failure text blamed an ambiguity that was not there. +public class PartitionMatcherStepBudgetTest { + /// The largest partition admitted by default; the repeated row object keeps the test's input allocation negligible. + private static final int NUM_ROWS = MatchLimits.DEFAULT_MAX_ROWS_IN_MATCH_PARTITION; + + /// `PATTERN (A+)` where `A` has no DEFINE, so it matches every row: no backtracking is possible. + private static final RowPattern A_PLUS = quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true); + private static final List ANY_A = List.of(anySymbol("A")); + + @Test + public void testDefaultBudgetAdmitsALinearMatchOverALargePartition() { + PartitionMatcher matcher = new PartitionMatcher(PatternToNfaCompiler.compile(A_PLUS), ANY_A, INPUT_SCHEMA, + MatchLimits.DEFAULT_MAX_STEPS_PER_MATCH_ATTEMPT); + + // The greedy quantifier consumes the whole partition, which is exactly what the partition row limit allows. + assertEquals(matcher.match(matchingRows(NUM_ROWS), 0, 1), NUM_ROWS); + assertTrue(matcher.getRetainedBacktrackingBytes() <= 32L * 1024 * 1024, + "One-million-row A+ must retain at most 32 MiB of choice/undo state, retained: " + + matcher.getRetainedBacktrackingBytes()); + } + + @Test + public void testExceedingTheBudgetReportsThePartitionSizeAndTheRowsConsumed() { + // Only the diagnostics change: a budget that a linear match cannot fit in still throws rather than truncating. + PartitionMatcher matcher = + new PartitionMatcher(PatternToNfaCompiler.compile(A_PLUS), ANY_A, INPUT_SCHEMA, 100L); + + QueryException exception = + expectThrows(QueryException.class, () -> matcher.match(matchingRows(1000), 0, 1)); + String message = exception.getMessage(); + assertTrue(message.contains("maximum of 100 pattern matching steps"), message); + // The partition size and the consumed row count are what distinguish a linear blowup from a backtracking one. + assertTrue(message.contains("of a 1000-row partition"), message); + assertTrue(message.contains("rows consumed so far"), message); + assertTrue(message.contains(QueryOptionKey.MAX_STEPS_PER_MATCH_ATTEMPT), message); + } + + @Test + public void testTerminationCallbackInterruptsTheTransitionLoop() { + AtomicInteger checks = new AtomicInteger(); + PartitionMatcher matcher = new PartitionMatcher(PatternToNfaCompiler.compile(A_PLUS), ANY_A, INPUT_SCHEMA, + Long.MAX_VALUE, () -> { + checks.incrementAndGet(); + throw QueryErrorCode.QUERY_CANCELLATION.asException("cancelled during MATCH_RECOGNIZE"); + }); + + QueryException exception = + expectThrows(QueryException.class, () -> matcher.match(matchingRows(10_000), 0, 1)); + assertEquals(exception.getErrorCode(), QueryErrorCode.QUERY_CANCELLATION); + assertEquals(checks.get(), 1); + } + + @Test + public void testRetainedBacktrackingStateHasAHardMemoryLimit() { + // 2 KiB blocks the first undo-log growth; 3 KiB admits it but blocks the subsequent choice-stack growth. + for (long maxRetainedBytes : List.of(2L * 1024, 3L * 1024)) { + PartitionMatcher matcher = new PartitionMatcher(PatternToNfaCompiler.compile(A_PLUS), ANY_A, INPUT_SCHEMA, + Long.MAX_VALUE, () -> { }, maxRetainedBytes); + + QueryException exception = + expectThrows(QueryException.class, () -> matcher.match(matchingRows(100), 0, 1)); + assertEquals(exception.getErrorCode(), QueryErrorCode.SERVER_RESOURCE_LIMIT_EXCEEDED); + assertTrue(exception.getMessage().contains("hard limit"), exception.getMessage()); + assertTrue(exception.getMessage().contains("retained pattern backtracking state"), exception.getMessage()); + assertTrue(matcher.getRetainedBacktrackingBytes() <= maxRetainedBytes, + "Matcher must reject growth before retaining more than its hard cap"); + } + } + + private static List matchingRows(int numRows) { + return Collections.nCopies(numRows, new Object[]{0, "A", 0}); + } +} diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompilerTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompilerTest.java new file mode 100644 index 000000000000..e07a97bacf0c --- /dev/null +++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/match/PatternToNfaCompilerTest.java @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.query.runtime.operator.match; + +import java.util.List; +import org.apache.pinot.query.planner.plannode.PatternSymbol; +import org.apache.pinot.query.planner.plannode.RowPattern; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.Transition; +import org.apache.pinot.query.runtime.operator.match.PatternNfa.TransitionKind; +import org.apache.pinot.spi.exception.QueryException; +import org.testng.annotations.Test; + +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.INPUT_SCHEMA; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.alternate; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.anySymbol; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.concat; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.labelSymbols; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.quantifier; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.rows; +import static org.apache.pinot.query.runtime.operator.match.MatchTestFixtures.symbol; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; + + +/// Tests the shape of the compiled automaton and, through [PartitionMatcher], the matching semantics that the +/// transition ordering is supposed to produce. +/// +/// Matching is asserted as `(endPosition, classification)` where the classification spells out which pattern +/// variable each matched row was mapped to, so a preference bug shows up as a different string rather than as a +/// different row count only. +public class PatternToNfaCompilerTest { + private static final long UNLIMITED_STEPS = Long.MAX_VALUE; + + @Test + public void testGreedyQuantifierPrefersTheLongestMatch() { + // PATTERN (A*) over AAA: greedy takes all three rows. + assertMatch(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), labelSymbols("A"), "AAA", "AAA"); + } + + @Test + public void testReluctantQuantifierPrefersTheShortestMatch() { + // PATTERN (A*?) over AAA: reluctant leaves the loop immediately, which is a legal empty match. + assertMatch(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, false), labelSymbols("A"), "AAA", ""); + } + + @Test + public void testReluctantQuantifierStillHonoursItsMinimum() { + // PATTERN (A+?) over AAA: reluctant, but one repetition is mandatory. + assertMatch(quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, false), labelSymbols("A"), "AAA", "A"); + } + + @Test + public void testGreedyQuantifierBacktracksWhenTheRestOfThePatternNeedsRows() { + // PATTERN (A* A) over AAA: the greedy loop must give one row back so the trailing A can match. + RowPattern pattern = + concat(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), symbol(1)); + // Two symbols, both matching the label A, so the classification shows where the loop stopped. + List symbols = List.of(MatchTestFixtures.labelSymbol("A", 0), aliasOfA()); + assertMatch(pattern, symbols, "AAA", "AAX"); + } + + @Test + public void testAlternationPrefersTheLeftmostBranch() { + // Both X and Y match the label A, so only the branch order decides. + List symbols = List.of(labelledAs("X", "A", 0), labelledAs("Y", "A", 1)); + assertMatch(alternate(symbol(0), symbol(1)), symbols, "A", "X"); + assertMatch(alternate(symbol(1), symbol(0)), symbols, "A", "Y"); + } + + @Test + public void testAlternationFallsThroughToALaterBranch() { + // PATTERN (A | B) over B: the first branch cannot match, the second one can. + assertMatch(alternate(symbol(0), symbol(1)), labelSymbols("A", "B"), "B", "B"); + } + + @Test + public void testAlternationBacktracksOutOfAPreferredBranchThatBlocksTheRestOfThePattern() { + // PATTERN ((A B | A) B): the leftmost branch matches the first two rows on its own, but then the trailing B has + // nothing left. Preference must not win over completeness, so the search has to unwind the preferred branch and + // come back through the second one. A preference bug that forgets to backtrack reports no match at all here. + RowPattern pattern = concat(alternate(concat(symbol(0), symbol(1)), symbol(0)), symbol(1)); + assertMatch(pattern, labelSymbols("A", "B"), "AB", "AB"); + // With one more B the preferred branch does complete, and it must be the one that wins. + assertMatch(pattern, labelSymbols("A", "B"), "ABB", "ABB"); + } + + @Test + public void testReluctantQuantifierStopsAtTheFirstRowThatLetsTheRestOfThePatternMatch() { + // PATTERN (A*? B) over A A B: reluctant prefers to leave the loop at once, but the trailing B forces it to take + // exactly as many rows as it must - no more. + RowPattern pattern = concat(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, false), symbol(1)); + assertMatch(pattern, labelSymbols("A", "B"), "AAB", "AAB"); + } + + @Test + public void testGreedyAndReluctantDifferOnlyInWhichMatchTheyPrefer() { + // PATTERN (A{1,3} A) over AAAA, with a second variable that also matches an A so the split is visible. + // Greedy gives the loop as many rows as it can while still letting the trailing A match; reluctant gives it the + // fewest. Both are legal matches of the same pattern, which is exactly why the preference order decides. + List symbols = List.of(MatchTestFixtures.labelSymbol("A", 0), aliasOfA()); + assertMatch(concat(quantifier(symbol(0), 1, 3, true), symbol(1)), symbols, "AAAA", "AAAX"); + assertMatch(concat(quantifier(symbol(0), 1, 3, false), symbol(1)), symbols, "AAAA", "AX"); + } + + @Test + public void testBoundedQuantifierRespectsItsUpperBound() { + // PATTERN (A{2,3}) over AAAA: greedy stops at the upper bound rather than consuming everything. + assertMatch(quantifier(symbol(0), 2, 3, true), labelSymbols("A"), "AAAA", "AAA"); + } + + @Test + public void testBoundedQuantifierRespectsItsLowerBound() { + // PATTERN (A{2,3}) over A: a single row is not enough, so there is no match at all. + assertNoMatch(quantifier(symbol(0), 2, 3, true), labelSymbols("A"), "A"); + } + + @Test + public void testExactQuantifier() { + assertMatch(quantifier(symbol(0), 2, 2, true), labelSymbols("A"), "AAAA", "AA"); + } + + @Test + public void testReluctantBoundedQuantifierStopsAtItsLowerBound() { + assertMatch(quantifier(symbol(0), 2, 3, false), labelSymbols("A"), "AAAA", "AA"); + } + + @Test + public void testBoundedQuantifierUsesCounterRegistersInsteadOfUnrollingStates() { + // The whole point of counter registers: a huge repetition bound must not blow up the automaton. + PatternNfa small = PatternToNfaCompiler.compile(quantifier(symbol(0), 1, 3, true)); + PatternNfa huge = PatternToNfaCompiler.compile(quantifier(symbol(0), 1, 100_000, true)); + assertEquals(huge.getNumStates(), small.getNumStates()); + assertEquals(huge.getNumCounters(), 1); + assertEquals(huge.getNumStates(), PatternToNfaCompiler.compile( + quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true)).getNumStates()); + } + + @Test + public void testNestedQuantifiersGetIndependentCounterRegisters() { + PatternNfa nfa = PatternToNfaCompiler.compile(quantifier(quantifier(symbol(0), 2, 2, true), 3, 3, true)); + assertEquals(nfa.getNumCounters(), 2); + // (A{2}){3} consumes exactly six rows: the inner register must be reset by each outer iteration. + assertMatch(quantifier(quantifier(symbol(0), 2, 2, true), 3, 3, true), labelSymbols("A"), "AAAAAAAA", "AAAAAA"); + } + + @Test + public void testGreedyLoopHeadListsRepeatBeforeExit() { + assertLoopHeadOrder(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), TransitionKind.REPEAT, + TransitionKind.EXIT_LOOP); + } + + @Test + public void testReluctantLoopHeadListsExitBeforeRepeat() { + assertLoopHeadOrder(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, false), TransitionKind.EXIT_LOOP, + TransitionKind.REPEAT); + } + + @Test + public void testCompiledTransitionListsAreImmutable() { + PatternNfa nfa = PatternToNfaCompiler.compile(symbol(0)); + List transitions = nfa.getState(nfa.getStartState()).getTransitions(); + + expectThrows(UnsupportedOperationException.class, transitions::clear); + assertEquals(transitions.size(), 1); + assertEquals(transitions.get(0).getKind(), TransitionKind.MATCH); + } + + @Test + public void testEmptyCycleGuardTerminatesOnAStarStar() { + // PATTERN ((A*)*) is the canonical non terminating pattern: the outer loop can iterate forever over an inner loop + // that consumes nothing. It must terminate, and still consume everything it can. + RowPattern pattern = quantifier(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), 0, + RowPattern.Quantifier.UNBOUNDED, true); + assertMatch(pattern, labelSymbols("A"), "AAA", "AAA"); + // With nothing to consume the same pattern still terminates, with an empty match. + assertMatch(pattern, labelSymbols("A"), "BBB", ""); + } + + @Test + public void testEmptyCycleGuardTerminatesOnAnOptionalBody() { + // PATTERN ((A?)+): the body may match zero rows, so the loop could spin forever. + RowPattern pattern = + quantifier(quantifier(symbol(0), 0, 1, true), 1, RowPattern.Quantifier.UNBOUNDED, true); + assertMatch(pattern, labelSymbols("A"), "AAB", "AA"); + assertMatch(pattern, labelSymbols("A"), "BBB", ""); + } + + @Test + public void testAnchorsAreRelativeToThePartition() { + // PATTERN (^ A): matches only when the scan starts at the first row of the partition. + RowPattern anchoredStart = concat(RowPattern.AnchorStart.INSTANCE, symbol(0)); + assertEquals(matchAt(anchoredStart, labelSymbols("A"), "AA", 0), 1); + assertEquals(matchAt(anchoredStart, labelSymbols("A"), "AA", 1), PartitionMatcher.NO_MATCH); + + // PATTERN (A $): matches only when the match ends at the last row of the partition. + RowPattern anchoredEnd = concat(symbol(0), RowPattern.AnchorEnd.INSTANCE); + assertEquals(matchAt(anchoredEnd, labelSymbols("A"), "AA", 0), PartitionMatcher.NO_MATCH); + assertEquals(matchAt(anchoredEnd, labelSymbols("A"), "AA", 1), 2); + } + + @Test + public void testSymbolWithoutDefinitionMatchesEveryRow() { + // SQL:2016: a PATTERN variable with no DEFINE entry has the condition TRUE. + assertMatch(quantifier(symbol(0), 1, RowPattern.Quantifier.UNBOUNDED, true), List.of(anySymbol("A")), "XYZ", "AAA"); + } + + @Test + public void testStepBudgetThrowsInsteadOfGivingUp() { + // An ambiguous pattern over a long partition: cut it off at a tiny budget and it must fail loudly, because + // returning the matches it happened to find would silently drop the rest. + RowPattern pattern = concat(quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), + quantifier(symbol(0), 0, RowPattern.Quantifier.UNBOUNDED, true), symbol(1)); + List symbols = List.of(MatchTestFixtures.labelSymbol("A", 0), labelledAs("Z", "Z", 1)); + PatternNfa nfa = PatternToNfaCompiler.compile(pattern); + PartitionMatcher matcher = new PartitionMatcher(nfa, symbols, INPUT_SCHEMA, 20); + QueryException exception = + expectThrows(QueryException.class, () -> matcher.match(rows("AAAAAAAAAAAAAAAAAAAA"), 0, 1)); + assertTrue(exception.getMessage().contains("maxStepsPerMatchAttempt"), exception.getMessage()); + } + + /// A second pattern variable that also matches the label `A`, used to observe where a greedy loop stopped. + private static PatternSymbol aliasOfA() { + return labelledAs("X", "A", 1); + } + + /// A pattern variable named `name` whose DEFINE matches rows whose label column equals `label`. + private static PatternSymbol labelledAs(String name, String label, int ordinal) { + PatternSymbol delegate = MatchTestFixtures.labelSymbol(label, ordinal); + return new PatternSymbol(name, delegate.getDefinition()); + } + + private static void assertLoopHeadOrder(RowPattern pattern, TransitionKind first, TransitionKind second) { + PatternNfa nfa = PatternToNfaCompiler.compile(pattern); + List loopHead = null; + for (PatternNfa.State state : nfa.getStates()) { + if (state.getTransitions().size() == 2 && state.getTransitions().stream() + .allMatch(t -> t.getKind() == TransitionKind.REPEAT || t.getKind() == TransitionKind.EXIT_LOOP)) { + loopHead = state.getTransitions(); + } + } + assertNotNull(loopHead, "No loop head found in " + nfa); + assertEquals(loopHead.get(0).getKind(), first); + assertEquals(loopHead.get(1).getKind(), second); + } + + /// Asserts that the preferred match starting at row 0 maps exactly the rows spelled out by + /// `expectedClassification`, whose i-th character is the name of the pattern variable the i-th row of the + /// match was mapped to. + private static void assertMatch(RowPattern pattern, List symbols, String input, + String expectedClassification) { + PartitionMatcher matcher = newMatcher(pattern, symbols); + List partitionRows = rows(input); + int endPos = matcher.match(partitionRows, 0, 1); + assertTrue(endPos != PartitionMatcher.NO_MATCH, "Expecting a match of '" + expectedClassification + "'"); + assertEquals(endPos, expectedClassification.length()); + StringBuilder classification = new StringBuilder(); + for (int i = 0; i < endPos; i++) { + classification.append(matcher.getTape().classifierAt(i)); + } + assertEquals(classification.toString(), expectedClassification); + } + + private static void assertNoMatch(RowPattern pattern, List symbols, String input) { + assertEquals(matchAt(pattern, symbols, input, 0), PartitionMatcher.NO_MATCH); + } + + private static int matchAt(RowPattern pattern, List symbols, String input, int startPos) { + return newMatcher(pattern, symbols).match(rows(input), startPos, 1); + } + + private static PartitionMatcher newMatcher(RowPattern pattern, List symbols) { + return new PartitionMatcher(PatternToNfaCompiler.compile(pattern), symbols, INPUT_SCHEMA, UNLIMITED_STEPS); + } +} diff --git a/pinot-query-runtime/src/test/resources/queries/MatchRecognize.json b/pinot-query-runtime/src/test/resources/queries/MatchRecognize.json new file mode 100644 index 000000000000..9af64a8cd4c4 --- /dev/null +++ b/pinot-query-runtime/src/test/resources/queries/MatchRecognize.json @@ -0,0 +1,93 @@ +{ + "match_recognize_v_shape": { + "tables": { + "ticker": { + "schema": [ + {"name": "symbol", "type": "STRING"}, + {"name": "seq", "type": "LONG"}, + {"name": "price", "type": "INT"} + ], + "inputs": [ + ["A", 1, 10], + ["A", 2, 8], + ["A", 3, 6], + ["A", 4, 9], + ["A", 5, 12], + ["A", 6, 7], + ["A", 7, 11], + ["B", 1, 5], + ["B", 2, 3], + ["B", 3, 8] + ] + } + }, + "queries": [ + { + "description": "V-shape detection with an explicit AFTER MATCH SKIP PAST LAST ROW: matches never overlap. H2 has no MATCH_RECOGNIZE, so the expected rows are hand-computed.", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES FIRST(DOWN.price) AS start_price, LAST(UP.price) AS end_price ONE ROW PER MATCH AFTER MATCH SKIP PAST LAST ROW PATTERN (DOWN+ UP+) DEFINE DOWN AS DOWN.price < PREV(DOWN.price), UP AS UP.price > PREV(UP.price)) AS mr", + "outputs": [ + ["A", 8, 12], + ["A", 7, 11], + ["B", 3, 8] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + }, + { + "description": "THE CRITICAL DEFAULT: omitting AFTER MATCH must behave as SKIP PAST LAST ROW (SQL:2016 / Trino / Snowflake / Oracle), NOT Calcite's internal SKIP TO NEXT ROW. Expected rows are byte-identical to the explicit PAST LAST ROW query above; if the default regressed to SKIP TO NEXT ROW this query would return the extra overlapping row [A, 6, 12].", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES FIRST(DOWN.price) AS start_price, LAST(UP.price) AS end_price ONE ROW PER MATCH PATTERN (DOWN+ UP+) DEFINE DOWN AS DOWN.price < PREV(DOWN.price), UP AS UP.price > PREV(UP.price)) AS mr", + "outputs": [ + ["A", 8, 12], + ["A", 7, 11], + ["B", 3, 8] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + }, + { + "description": "Explicit AFTER MATCH SKIP TO NEXT ROW produces overlapping matches. The extra row [A, 6, 12] is what an engine that silently kept Calcite's default would wrongly return for the query above, so this pins that the two modes really do differ.", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES FIRST(DOWN.price) AS start_price, LAST(UP.price) AS end_price ONE ROW PER MATCH AFTER MATCH SKIP TO NEXT ROW PATTERN (DOWN+ UP+) DEFINE DOWN AS DOWN.price < PREV(DOWN.price), UP AS UP.price > PREV(UP.price)) AS mr", + "outputs": [ + ["A", 8, 12], + ["A", 6, 12], + ["A", 7, 11], + ["B", 3, 8] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + }, + { + "description": "MATCH_NUMBER() restarts per partition, CLASSIFIER() reports the label of the final row of the match, and a single-variable aggregate counts only the rows bound to that variable.", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES MATCH_NUMBER() AS mno, CLASSIFIER() AS cls, COUNT(DOWN.price) AS down_cnt ONE ROW PER MATCH PATTERN (DOWN+ UP+) DEFINE DOWN AS DOWN.price < PREV(DOWN.price), UP AS UP.price > PREV(UP.price)) AS mr", + "outputs": [ + ["A", 1, "UP", 2], + ["A", 2, "UP", 1], + ["B", 1, "UP", 1] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + }, + { + "description": "Start anchor plus a pattern variable that has no DEFINE entry (SQL:2016 says it matches every row). The anchor is load-bearing: without it partition A would also yield the later match [A, 7, 7] starting at seq=5.", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES FIRST(DIP.price) AS first_dip, LAST(DIP.price) AS last_dip ONE ROW PER MATCH PATTERN (^ START DIP+) DEFINE DIP AS DIP.price < PREV(DIP.price)) AS mr", + "outputs": [ + ["A", 8, 6], + ["B", 3, 3] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + }, + { + "description": "Same pattern without the start anchor, to prove the anchor above is actually enforced rather than silently dropped: partition A now also matches the later dip at seq=5..6.", + "sql": "SELECT * FROM {ticker} MATCH_RECOGNIZE (PARTITION BY symbol ORDER BY seq MEASURES FIRST(DIP.price) AS first_dip, LAST(DIP.price) AS last_dip ONE ROW PER MATCH PATTERN (START DIP+) DEFINE DIP AS DIP.price < PREV(DIP.price)) AS mr", + "outputs": [ + ["A", 8, 6], + ["A", 7, 7], + ["B", 3, 3] + ], + "ignoreV2Optimizer": true, + "ignoreLiteMode": true + } + ] + } +} diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java index 0ef4615ba200..e1868b9a29bd 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java @@ -813,6 +813,16 @@ public static class QueryOptionKey { // un-upgraded server cannot honor, so only enable it once all servers support it. // NOTE: This is a no-op under usePhysicalOptimizer (the v2 path does not go through RelToPlanNodeConverter). public static final String UNNEST_COLUMN_PRUNING = "unnestColumnPruning"; + /// A MATCH_RECOGNIZE partition is evaluated as a whole on a single worker, so a query without a PARTITION BY + /// clause hashes on zero keys and routes the entire table to one worker. The multi-stage planner rejects that + /// plan by default; set this to true to accept single-worker execution (fine for small tables, or when the + /// WHERE clause already narrows the input down to one logical partition). + public static final String ALLOW_MATCH_RECOGNIZE_WITHOUT_PARTITION_BY = "allowMatchRecognizeWithoutPartitionBy"; + /// Maximum number of rows that a single MATCH_RECOGNIZE partition may buffer on a worker. This is a + /// partition limit, not a per-match limit: all rows of a partition must be available before matching starts. + public static final String MAX_ROWS_IN_MATCH_PARTITION = "maxRowsInMatchPartition"; + /// Maximum number of automaton transitions explored for one MATCH_RECOGNIZE start position. + public static final String MAX_STEPS_PER_MATCH_ATTEMPT = "maxStepsPerMatchAttempt"; /// When set to true, the broker uses the long-lived `SubmitWithStream` bidi RPC to dispatch the query, /// receiving stage stats out-of-band as `OpChainComplete` messages instead of via mailbox EOS. The /// broker awaits stats completion as soon as the receiving mailbox finishes (early completion), bounded by