From a5b8b017b295512a9b07a382a02dc8f22f2055c7 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Thu, 17 Sep 2026 06:46:31 +0300 Subject: [PATCH] [#1059] Give no key up on import or rebuild under an index-entry-limit of 0 An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"), and the live index honours it: DefaultIndex.computeEntryIDSet gives a key up only while the limit is above 0. The two collectors of the on-disk merge importer compared the value as a count instead - 0 < 0 never accepted an entry ID and 0 >= 0 always answered the undefined set - so every key an import-ldif or a rebuild-index wrote under that setting was undefined, in an index the import then marked trusted. Every search through such an index was unindexed, and refused to a client without the unindexed-search privilege. The importer this one replaced (OPENDJ-2016) mapped 0 to Integer.MAX_VALUE; the collectors now do the same. backendstat show-index-status read the value the same way and counted every key of such an index as over 95% of its limit; an index with no limit has no key near it. Indexes imported or rebuilt under index-entry-limit: 0 before this change hold every key undefined and have to be rebuilt once. Part of #1059: whether a changed limit needs a rebuild is decided by the same comparison in DefaultIndex.setIndexEntryLimit, which #997 moves to AttributeIndex.planIndexUpdates; that part follows once #997 is merged. --- .../backends/pluggable/BackendStat.java | 11 +- .../pluggable/OnDiskMergeImporter.java | 15 ++- .../backends/pluggable/BackendStatTest.java | 43 +++++++ .../pluggable/OnDiskMergeImporterTest.java | 112 ++++++++++++++++ .../PluggableBackendImplTestCase.java | 120 +++++++++++++++++- 5 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/BackendStatTest.java diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java index dbee29479d..4ee981cde7 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendStat.java @@ -1100,6 +1100,15 @@ private void appendStatsNoData(TableBuilder builder, int columns) } } + /** + * Whether a key holding this many entries has come near the entry limit of its index. An + * index-entry-limit of 0 is no limit at all, and no key is near it. + */ + static boolean nearLimit(long size, long entryLimit) + { + return entryLimit > 0 && size >= entryLimit * 0.8; + } + private void appendIndexStats(final TableBuilder builder, EntryContainer ec, final Index index, final Map undefinedKeys) { @@ -1136,7 +1145,7 @@ public Void run(ReadableTransaction txn) throws Exception if (entryIDSet.isDefined()) { - if (entryIDSet.size() >= entryLimit * 0.8) + if (nearLimit(entryIDSet.size(), entryLimit)) { if (entryIDSet.size() >= entryLimit * 0.95) { diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java index efb49c5297..cd8bb92e71 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java @@ -3552,6 +3552,17 @@ public V merge(V latestValue) } } + /** + * The number of entry IDs at which the collectors give a key up. An index-entry-limit of 0 is no + * limit at all ("For no limit, use 0 for the value"), as the live index takes it; compared as a + * count it would have every key given up. + */ + private static int entryLimitOf(DefaultIndex index) + { + final int indexEntryLimit = index.getIndexEntryLimit(); + return indexEntryLimit > 0 ? indexEntryLimit : Integer.MAX_VALUE; + } + /** * {@link Collector} that accepts encoded {@link EntryIDSet} objects and * produces a {@link ByteString} representing the merged {@link EntryIDSet}. @@ -3564,7 +3575,7 @@ static final class EntryIDsCollector implements Collector EntryIDsCollector(DefaultIndex index) { this.index = index; - this.indexLimit = index.getIndexEntryLimit(); + this.indexLimit = entryLimitOf(index); } @Override @@ -3638,7 +3649,7 @@ static final class EntryIDSetsCollector implements Collector source = cursorOf( + Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))), + Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + + Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(2))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(3))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(4))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(5))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(6))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(7))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(8))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(9))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))), + + Pair.of("key4", EntryIDSet.CODEC_V2.encode(newDefinedSet(10))), + Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); + + final SequentialCursor expected = cursorOf( + Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), + Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))), + Pair.of("key4", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); + + final SequentialCursor result = + new CollectorCursor<>(source, new EntryIDSetsCollector(new DummyIndex(0))); + + assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); + } + + /** The phase-one collector gives a key up once it holds as many entry IDs as the limit allows. */ + @Test + public void testEntryIDsCollector() + { + final MeteredCursor source = cursorOf( + Pair.of("key1", entryID(2)), + Pair.of("key1", entryID(1)), + + Pair.of("key2", entryID(1)), + + Pair.of("key3", entryID(1)), + Pair.of("key3", entryID(2)), + Pair.of("key3", entryID(3)), + Pair.of("key3", entryID(4)), + Pair.of("key3", entryID(5)), + Pair.of("key3", entryID(6)), + Pair.of("key3", entryID(7)), + Pair.of("key3", entryID(8)), + Pair.of("key3", entryID(9)), + Pair.of("key3", entryID(10))); + + final SequentialCursor expected = cursorOf( + Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), + Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newUndefinedSet()))); + + final SequentialCursor result = + new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(10))); + + assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); + } + + /** Under an index-entry-limit of 0 the phase-one collector gives no key up either (#1059). */ + @Test + public void testEntryIDsCollectorGivesNoKeyUpUnderNoLimit() + { + final MeteredCursor source = cursorOf( + Pair.of("key1", entryID(2)), + Pair.of("key1", entryID(1)), + + Pair.of("key2", entryID(1)), + + Pair.of("key3", entryID(1)), + Pair.of("key3", entryID(2)), + Pair.of("key3", entryID(3)), + Pair.of("key3", entryID(4)), + Pair.of("key3", entryID(5)), + Pair.of("key3", entryID(6)), + Pair.of("key3", entryID(7)), + Pair.of("key3", entryID(8)), + Pair.of("key3", entryID(9)), + Pair.of("key3", entryID(10))); + + final SequentialCursor expected = cursorOf( + Pair.of("key1", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2))), + Pair.of("key2", EntryIDSet.CODEC_V2.encode(newDefinedSet(1))), + Pair.of("key3", EntryIDSet.CODEC_V2.encode(newDefinedSet(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)))); + + final SequentialCursor result = + new CollectorCursor<>(source, new EntryIDsCollector(new DummyIndex(0))); + + assertThat(toPairs(result)).containsExactlyElementsOf(toPairs(expected)); + } + + /** An entry ID the way phase one buffers it: the bare ID, not an {@link EntryIDSet}. */ + private static ByteString entryID(long id) + { + return new EntryID(id).toByteString(); + } + @Test public void testUniqueValueCollectorAcceptUniqueValues() { diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java index 6706f7b402..6adabad999 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java @@ -93,6 +93,7 @@ public abstract class PluggableBackendImplTestCase extends DirectoryServerTestCase { private BackendImpl backend; + private C backendCfg; private List topEntries; private List entries; private List workEntries; @@ -142,7 +143,7 @@ public void setUp() throws Exception testBaseDN = DN.valueOf("dc=test,dc=com"); - C backendCfg = createBackendCfg(); + backendCfg = createBackendCfg(); when(backendCfg.dn()).thenReturn(testBaseDN); when(backendCfg.getBaseDN()).thenReturn(newTreeSet(testBaseDN)); when(backendCfg.listBackendIndexes()).thenReturn(backendIndexes.keySet().toArray(new String[0])); @@ -1159,6 +1160,123 @@ public void run(WriteableTransaction txn) throws Exception assertThat(backend.verifyBackend(config)).isEqualTo(0); } + /** + * An index-entry-limit of 0 is no limit at all ("For no limit, use 0 for the value"), and the live + * index honours it. The importer compared it as the smallest limit there is and wrote every key of + * an import undefined: the index was then trusted, and every search through it unindexed (#1059). + */ + @Test + public void testImportUnderNoIndexEntryLimitKeepsEveryKey() throws Exception + { + final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn"); + when(snIndexCfg.getIndexEntryLimit()).thenReturn(0); + try + { + final byte[] ldif = exportLDIF(); + backend.finalizeBackend(); + importLDIF(ldif); + backend.openBackend(); + + final Map keys = indexKeys("sn"); + assertThat(keys).as("the keys the sn indexes hold").isNotEmpty(); + assertThat(keys).as("a key the import gave up under no limit").doesNotContainValue(false); + } + finally + { + when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000); + reopenBackend(); + } + } + + /** A rebuild writes its keys the way an import does: under no limit it gives none of them up either. */ + @Test + public void testRebuildUnderNoIndexEntryLimitKeepsEveryKey() throws Exception + { + final BackendIndexCfg snIndexCfg = backendCfg.getBackendIndex("sn"); + when(snIndexCfg.getIndexEntryLimit()).thenReturn(0); + try + { + final RebuildConfig rebuildConfig = new RebuildConfig(); + rebuildConfig.setBaseDN(testBaseDN); + rebuildConfig.addRebuildIndex("sn"); + backend.closeBackend(); + backend.rebuildBackend(rebuildConfig, TestCaseUtils.getServerContext()); + backend.openBackend(); + + final Map keys = indexKeys("sn"); + assertThat(keys).as("the keys the sn indexes hold").isNotEmpty(); + assertThat(keys).as("a key the rebuild gave up under no limit").doesNotContainValue(false); + } + finally + { + when(snIndexCfg.getIndexEntryLimit()).thenReturn(4000); + reopenBackend(); + } + } + + private byte[] exportLDIF() throws Exception + { + final ByteArrayOutputStream ldif = new ByteArrayOutputStream(); + try (LDIFExportConfig exportConfig = new LDIFExportConfig(ldif)) + { + exportConfig.setIncludeOperationalAttributes(true); + backend.exportLDIF(exportConfig); + } + return ldif.toByteArray(); + } + + /** Imports the LDIF into the cleared backend, which the caller has finalized and opens again afterwards. */ + private void importLDIF(byte[] ldif) throws Exception + { + final ByteArrayOutputStream rejectedEntries = new ByteArrayOutputStream(); + try (LDIFImportConfig importConfig = new LDIFImportConfig(new ByteArrayInputStream(ldif))) + { + importConfig.setClearBackend(true); + importConfig.writeRejectedEntries(rejectedEntries); + importConfig.setIncludeBranches(Collections.singleton(testBaseDN)); + importConfig.setThreadCount(0); + backend.importLDIF(importConfig, TestCaseUtils.getServerContext()); + } + assertEquals(rejectedEntries.size(), 0, "No entries should be rejected. Content was:\n" + rejectedEntries); + } + + /** Every key of every index of the attribute, and whether the index still holds its entries. */ + private Map indexKeys(String attributeName) throws Exception + { + final AttributeType attributeType = TestCaseUtils.getServerContext().getSchema().getAttributeType(attributeName); + final EntryContainer entryContainer = backend.getRootContainer().getEntryContainer(testBaseDN); + final AttributeIndex attributeIndex = entryContainer.getAttributeIndex(attributeType); + return backend.getRootContainer().getStorage().read(new ReadOperation>() + { + @Override + public Map run(ReadableTransaction txn) throws Exception + { + final Map keys = new TreeMap<>(); + for (AttributeIndex.MatchingRuleIndex index : attributeIndex.getNameToIndexes().values()) + { + try (Cursor cursor = index.openCursor(txn)) + { + while (cursor.next()) + { + keys.put(index.getName() + " " + cursor.getKey().toHexString(), cursor.getValue().isDefined()); + } + } + } + return keys; + } + }); + } + + /** Opens the backend afresh, so that its indexes hold the configuration the other tests expect. */ + private void reopenBackend() throws Exception + { + if (backend.getRootContainer() != null) + { + backend.finalizeBackend(); + } + backend.openBackend(); + } + @Test public void testVerifyID2ChildrenCount() throws Exception {