diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java index 3308566bd712..12091b6b13e5 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/reader/impl/FixedBitIntReader.java @@ -106,7 +106,8 @@ public static FixedBitIntReader getReader(PinotDataBuffer dataBuffer, int numBit case 31: return new Bit31Reader(dataBuffer); default: - throw new IllegalStateException(); + throw new IllegalStateException("Illegal number of bits per value: " + numBitsPerValue + ", must be within 1 " + + "and 31"); } } diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java index e686d40e763a..e6399bea7487 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; import javax.annotation.Nullable; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.ex.ConfigurationException; @@ -256,16 +257,18 @@ private List columnMinMaxValueUpdates() { } private boolean needProcessStarTrees() { + SegmentMetadataImpl segmentMetadata = _segmentDirectory.getSegmentMetadata(); + List starTreeMetadataList = segmentMetadata.getStarTreeV2MetadataList(); // Check if there is need to create/modify/remove star-trees. if (!_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { - return false; + // Star-trees left unreadable by a column encoding change are still removed, see processStarTrees(). + return starTreeMetadataList != null && !StarTreeBuilderUtils.findUnloadableDimensions(starTreeMetadataList, + segmentMetadata).isEmpty(); } - SegmentMetadataImpl segmentMetadata = _segmentDirectory.getSegmentMetadata(); List starTreeBuilderConfigs = StarTreeBuilderUtils.generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), segmentMetadata); - List starTreeMetadataList = segmentMetadata.getStarTreeV2MetadataList(); // There are existing star-trees, but if they match the builder configs exactly, // then there is no need to generate the star-trees @@ -358,19 +361,35 @@ private void removeMultiColumnTextIndex(File indexDir) private boolean processStarTrees(File indexDir, @Nullable SegmentOperationsThrottlerSet segmentOperationsThrottlerSet) throws Exception { + SegmentMetadataImpl segmentMetadata = _segmentDirectory.getSegmentMetadata(); + String segmentName = segmentMetadata.getName(); + List starTreeMetadataList = segmentMetadata.getStarTreeV2MetadataList(); + if (!_indexLoadingConfig.isEnableDynamicStarTreeCreation()) { - return false; + // A star-tree whose dimension column is no longer dictionary-encoded (e.g. because the column was added to + // 'noDictionaryColumns' and re-encoded by the forward index handler above) cannot be read, and fails the whole + // segment load. Drop it even here: removing star-trees only deletes files, so unlike rebuilding them it is + // cheap enough to do with dynamic star-tree creation disabled. When it is enabled the star-trees are rebuilt by + // the regular flow below, because their split order no longer matches the builder configs. + Set unloadableDimensions = starTreeMetadataList != null + ? StarTreeBuilderUtils.findUnloadableDimensions(starTreeMetadataList, segmentMetadata) + : Set.of(); + if (unloadableDimensions.isEmpty()) { + return false; + } + LOGGER.warn("Removing star-trees from segment: {} because dimension columns: {} are no longer " + + "dictionary-encoded. Enable dynamic star-tree creation to have them rebuilt", segmentName, + unloadableDimensions); + StarTreeBuilderUtils.removeStarTrees(indexDir); + return true; } - SegmentMetadataImpl segmentMetadata = _segmentDirectory.getSegmentMetadata(); - String segmentName = segmentMetadata.getName(); List starTreeBuilderConfigs = StarTreeBuilderUtils.generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(), _indexLoadingConfig.isEnableDefaultStarTree(), segmentMetadata); boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty(); boolean shouldRemoveStarTree = false; - List starTreeMetadataList = segmentMetadata.getStarTreeV2MetadataList(); if (starTreeMetadataList != null) { // There are existing star-trees if (!shouldGenerateStarTree) { diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java index 72e73b4cfd8c..465a5737f400 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java @@ -28,7 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import javax.annotation.Nullable; import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.io.FileUtils; @@ -36,6 +38,7 @@ import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig; import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.ColumnMetadata; import org.apache.pinot.segment.spi.Constants; import org.apache.pinot.segment.spi.SegmentMetadata; import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair; @@ -283,6 +286,37 @@ public static boolean shouldModifyExistingStarTrees(List findUnloadableDimensions(List metadataList, + SegmentMetadata segmentMetadata) { + Set dimensions = new TreeSet<>(); + for (StarTreeV2Metadata starTreeMetadata : metadataList) { + String dimension = findUnloadableDimension(starTreeMetadata, segmentMetadata); + if (dimension != null) { + dimensions.add(dimension); + } + } + return dimensions; + } + /// Returns `true` if the given star-tree builder configs are equal, `false` otherwise. public static boolean areStarTreeBuilderConfigListsEqual(List builderConfig1, List builderConfig2) { diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java index 4e6ab15b49bb..07e1a8c7d1f8 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/store/StarTreeLoaderUtils.java @@ -27,6 +27,7 @@ import org.apache.pinot.segment.local.segment.index.forward.ForwardIndexReaderFactory; import org.apache.pinot.segment.local.segment.index.readers.forward.FixedBitSVForwardIndexReaderV2; import org.apache.pinot.segment.local.startree.OffHeapStarTree; +import org.apache.pinot.segment.local.startree.StarTreeBuilderUtils; import org.apache.pinot.segment.spi.ColumnMetadata; import org.apache.pinot.segment.spi.datasource.DataSource; import org.apache.pinot.segment.spi.index.StandardIndexes; @@ -42,10 +43,14 @@ import org.apache.pinot.spi.data.FieldSpec; import org.apache.pinot.spi.data.FieldSpec.DataType; import org.apache.pinot.spi.data.MetricFieldSpec; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /// The `StarTreeLoaderUtils` class provides utility methods to load star-tree indexes. public class StarTreeLoaderUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(StarTreeLoaderUtils.class); + private StarTreeLoaderUtils() { } @@ -57,11 +62,23 @@ public static List loadStarTreeV2(SegmentDirectory.Reader segmentRea int numStarTrees = starTreeMetadataList.size(); List starTrees = new ArrayList<>(numStarTrees); for (int i = 0; i < numStarTrees; i++) { + StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(i); + + // A star-tree is unreadable once one of its dimension columns loses its dictionary, which happens when the + // column is moved to 'noDictionaryColumns' without the star-tree being rebuilt. Skip it instead of failing the + // whole segment load. SegmentPreProcessor normally removes such star-trees, so reaching this point means the + // pre-processing was skipped for this segment. + String unloadableDimension = StarTreeBuilderUtils.findUnloadableDimension(starTreeMetadata, segmentMetadata); + if (unloadableDimension != null) { + LOGGER.warn("Skipping star-tree: {} in segment: {} because dimension column: {} is no longer " + + "dictionary-encoded", i, segmentMetadata.getName(), unloadableDimension); + continue; + } + SegmentDirectory.Reader indexReader = segmentReader.getStarTreeIndexReader(i); // Load star-tree index StarTree starTree = new OffHeapStarTree(indexReader.getIndexFor(String.valueOf(i), StandardIndexes.inverted())); - StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(i); int numDocs = starTreeMetadata.getNumDocs(); Map dataSourceMap = new HashMap<>(); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java index c7272504a375..95ea45e9434b 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java @@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.pinot.segment.local.PinotBuffersAfterClassCheckRule; +import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.io.util.PinotDataBitSet; import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -46,9 +47,11 @@ import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode; import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader; import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory; +import org.apache.pinot.segment.local.startree.StarTreeBuilderUtils; import org.apache.pinot.segment.local.utils.SegmentOperationsThrottler; import org.apache.pinot.segment.local.utils.SegmentOperationsThrottlerSet; import org.apache.pinot.segment.spi.ColumnMetadata; +import org.apache.pinot.segment.spi.ImmutableSegment; import org.apache.pinot.segment.spi.V1Constants; import org.apache.pinot.segment.spi.compression.ChunkCompressionType; import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig; @@ -61,6 +64,7 @@ import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl; import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader; import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair; +import org.apache.pinot.segment.spi.index.startree.StarTreeV2; import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata; import org.apache.pinot.segment.spi.store.SegmentDirectory; import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths; @@ -81,6 +85,7 @@ import org.apache.pinot.spi.data.Schema; import org.apache.pinot.spi.data.readers.GenericRow; import org.apache.pinot.spi.utils.ByteArray; +import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.spi.utils.builder.TableConfigBuilder; import org.testng.annotations.AfterMethod; @@ -2025,6 +2030,245 @@ public void testStarTreeCreationWithDictionaryChanges() } } + /// A star-tree dimension column that is moved to 'noDictionaryColumns' without the star-tree being rebuilt leaves + /// the star-tree unreadable: its dimension forward index stores dictionary ids in a fixed-bit encoding whose width + /// is read from the main column metadata, which is now raw. The stale star-tree must be dropped so the segment + /// stays loadable, even when dynamic star-tree creation is disabled. + @Test + public void testStarTreeDimensionConvertedToNoDictionary() + throws Exception { + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("stringCol", DataType.STRING) + .addMetric("longCol", DataType.LONG) + .build(); + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__longCol"), null, 1000))); + buildStarTreeTestSegment(tableConfig, schema); + + // Drift the config: the star-tree dimension is moved to noDictionaryColumns and the star-tree config is dropped, + // while dynamic star-tree creation stays disabled. + indexingConfig.setNoDictionaryColumns(List.of("stringCol")); + indexingConfig.setStarTreeIndexConfigs(null); + indexingConfig.setEnableDynamicStarTreeCreation(false); + IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(tableConfig, schema); + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap); + SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) { + assertTrue(processor.needProcess()); + processor.process(SEGMENT_OPERATIONS_THROTTLER); + } + assertSegmentLoadsWithoutStarTree(indexLoadingConfig); + + // The stale star-tree is gone, so there is nothing left to process + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap); + SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) { + assertFalse(processor.needProcess()); + } + } + + /// Same drift as [#testStarTreeDimensionConvertedToNoDictionary()], but the dict-to-raw conversion has already been + /// persisted by an earlier pre-processing round, so the segment on disk is already inconsistent and nothing else + /// needs updating. Pre-processing must still detect and repair it. + @Test + public void testStarTreeDimensionAlreadyConvertedToNoDictionary() + throws Exception { + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("stringCol", DataType.STRING) + .addMetric("longCol", DataType.LONG) + .build(); + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__longCol"), null, 1000))); + buildStarTreeTestSegment(tableConfig, schema); + + // Convert the dimension column to raw while leaving the star-tree in place, reproducing the state an earlier + // pre-processing round leaves behind. + indexingConfig.setNoDictionaryColumns(List.of("stringCol")); + IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(tableConfig, schema); + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + new ForwardIndexHandler(segmentDirectory, indexLoadingConfig).updateIndices(segmentDirectory.createWriter()); + } + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + assertFalse(segmentDirectory.getSegmentMetadata().getColumnMetadataFor("stringCol").hasDictionary()); + assertNotNull(segmentDirectory.getSegmentMetadata().getStarTreeV2MetadataList()); + } + + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap); + SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) { + assertTrue(processor.needProcess()); + processor.process(SEGMENT_OPERATIONS_THROTTLER); + } + assertSegmentLoadsWithoutStarTree(indexLoadingConfig); + } + + /// The loader must not fail the whole segment over a stale star-tree even when pre-processing never gets a chance to + /// repair it, e.g. because it is skipped for the table. + @Test + public void testStarTreeDimensionConvertedToNoDictionaryWithoutPreprocess() + throws Exception { + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("stringCol", DataType.STRING) + .addMetric("longCol", DataType.LONG) + .build(); + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__longCol"), null, 1000))); + buildStarTreeTestSegment(tableConfig, schema); + + indexingConfig.setNoDictionaryColumns(List.of("stringCol")); + IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(tableConfig, schema); + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + new ForwardIndexHandler(segmentDirectory, indexLoadingConfig).updateIndices(segmentDirectory.createWriter()); + } + + // The stale star-tree is still in the segment, but it must be skipped rather than fail the load + ImmutableSegment segment = ImmutableSegmentLoader.load(INDEX_DIR, indexLoadingConfig, false); + try { + assertEquals(segment.getSegmentMetadata().getTotalDocs(), 5); + assertTrue(segment.getStarTrees() == null || segment.getStarTrees().isEmpty()); + } finally { + segment.destroy(); + } + } + + /// With dynamic star-tree creation enabled, the stale star-tree is not just dropped but rebuilt from the current + /// config, which no longer splits on the re-encoded column. + @Test + public void testStarTreeDimensionConvertedToNoDictionaryWithDynamicCreation() + throws Exception { + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("stringCol", DataType.STRING) + .addSingleValueDimension("intCol", DataType.INT) + .addMetric("longCol", DataType.LONG) + .build(); + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("stringCol", "intCol"), null, List.of("SUM__longCol"), null, 1000))); + buildStarTreeTestSegment(tableConfig, schema); + + // 'stringCol' becomes raw and drops out of the split order, and the star-tree is rebuilt on 'intCol' alone + indexingConfig.setNoDictionaryColumns(List.of("stringCol")); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("intCol"), null, List.of("SUM__longCol"), null, 1000))); + indexingConfig.setEnableDynamicStarTreeCreation(true); + IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(tableConfig, schema); + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap); + SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) { + assertTrue(processor.needProcess()); + processor.process(SEGMENT_OPERATIONS_THROTTLER); + } + + ImmutableSegment segment = ImmutableSegmentLoader.load(INDEX_DIR, indexLoadingConfig, false); + try { + List starTrees = segment.getStarTrees(); + assertNotNull(starTrees); + assertEquals(starTrees.size(), 1); + assertEquals(starTrees.get(0).getMetadata().getDimensionsSplitOrder(), List.of("intCol")); + } finally { + segment.destroy(); + } + } + + /// Apache Pinot PR #19153 added star-tree support for dimensions stored as a `RAW` forward index with a separated + /// dictionary. Such a column still has a dictionary, so its star-tree stays readable and must NOT be treated as + /// stale: pre-processing has to flip the forward index to raw, keep the dictionary, and leave the star-tree alone. + @Test + public void testStarTreeDimensionConvertedToRawWithSeparatedDictionary() + throws Exception { + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build(); + Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("stringCol", DataType.STRING) + .addMetric("longCol", DataType.LONG) + .build(); + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + indexingConfig.setStarTreeIndexConfigs( + List.of(new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__longCol"), null, 1000))); + buildStarTreeTestSegment(tableConfig, schema); + + // Keep the star-tree config, but store the dimension as RAW forward index with the dictionary kept alongside + ObjectNode indexes = JsonUtils.newObjectNode(); + ObjectNode forwardConfig = JsonUtils.newObjectNode(); + forwardConfig.put("encodingType", "RAW"); + indexes.set("forward", forwardConfig); + ObjectNode dictionaryConfig = JsonUtils.newObjectNode(); + dictionaryConfig.put("disabled", false); + indexes.set("dictionary", dictionaryConfig); + tableConfig.setFieldConfigList(List.of( + new FieldConfig.Builder("stringCol").withEncodingType(FieldConfig.EncodingType.RAW) + .withIndexes(indexes) + .build())); + indexingConfig.setEnableDynamicStarTreeCreation(false); + IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(tableConfig, schema); + + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap); + SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig)) { + processor.process(SEGMENT_OPERATIONS_THROTTLER); + } + + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + SegmentMetadataImpl segmentMetadata = segmentDirectory.getSegmentMetadata(); + ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor("stringCol"); + assertEquals(columnMetadata.getForwardIndexEncoding(), FieldConfig.EncodingType.RAW); + assertTrue(columnMetadata.hasDictionary()); + // The star-tree is still loadable, so it must be left in place + assertNotNull(segmentMetadata.getStarTreeV2MetadataList()); + assertTrue(StarTreeBuilderUtils.findUnloadableDimensions(segmentMetadata.getStarTreeV2MetadataList(), + segmentMetadata).isEmpty()); + } + + ImmutableSegment segment = ImmutableSegmentLoader.load(INDEX_DIR, indexLoadingConfig, false); + try { + List starTrees = segment.getStarTrees(); + assertNotNull(starTrees); + assertEquals(starTrees.size(), 1); + assertEquals(starTrees.get(0).getMetadata().getDimensionsSplitOrder(), List.of("stringCol")); + assertNotNull(segment.getDataSource("stringCol").getDictionary()); + } finally { + segment.destroy(); + } + } + + private void buildStarTreeTestSegment(TableConfig tableConfig, Schema schema) + throws Exception { + FileUtils.deleteQuietly(TEMP_DIR); + SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema); + config.setInstanceType(InstanceType.SERVER); + config.setOutDir(TEMP_DIR.getAbsolutePath()); + config.setSegmentName(SEGMENT_NAME); + + String[] stringValues = {"A", "C", "B", "C", "D"}; + long[] longValues = {2, 1, 2, 3, 4}; + List rows = new ArrayList<>(stringValues.length); + for (int i = 0; i < stringValues.length; i++) { + GenericRow row = new GenericRow(); + row.putValue("stringCol", stringValues[i]); + row.putValue("intCol", i % 3); + row.putValue("longCol", longValues[i]); + rows.add(row); + } + + SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); + driver.init(config, new GenericRowRecordReader(rows)); + driver.build(); + + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + assertNotNull(segmentDirectory.getSegmentMetadata().getStarTreeV2MetadataList()); + } + } + + private void assertSegmentLoadsWithoutStarTree(IndexLoadingConfig indexLoadingConfig) + throws Exception { + try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap)) { + assertNull(segmentDirectory.getSegmentMetadata().getStarTreeV2MetadataList()); + } + ImmutableSegment segment = ImmutableSegmentLoader.load(INDEX_DIR, indexLoadingConfig, false); + try { + assertEquals(segment.getSegmentMetadata().getTotalDocs(), 5); + assertTrue(segment.getStarTrees() == null || segment.getStarTrees().isEmpty()); + } finally { + segment.destroy(); + } + } + @Test public void testStarTreeCreationWithInvalidFunctionColumnPair() throws Exception {