This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 91156de091 #12635 Bug Fix createDictionaryForColumn does not take
into account inverted index (#13048)
91156de091 is described below
commit 91156de091f1a5d8cda2198d082fa2a8e8480827
Author: Chaitanya Deepthi <[email protected]>
AuthorDate: Tue May 7 17:17:51 2024 -0400
#12635 Bug Fix createDictionaryForColumn does not take into account
inverted index (#13048)
---
.../creator/impl/SegmentColumnarIndexCreator.java | 6 ++++--
.../index/dictionary/DictionaryIndexType.java | 4 ++++
.../index/loader/ForwardIndexHandlerTest.java | 25 ++++++++++++++++++++++
3 files changed, 33 insertions(+), 2 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
index 168490635a..1483191b7f 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentColumnarIndexCreator.java
@@ -141,8 +141,10 @@ public class SegmentColumnarIndexCreator implements
SegmentCreator {
ColumnIndexCreationInfo columnIndexCreationInfo =
indexCreationInfoMap.get(columnName);
Preconditions.checkNotNull(columnIndexCreationInfo, "Missing index
creation info for column: %s", columnName);
boolean dictEnabledColumn =
createDictionaryForColumn(columnIndexCreationInfo, segmentCreationSpec,
fieldSpec);
- Preconditions.checkState(dictEnabledColumn ||
!originalConfig.getConfig(StandardIndexes.inverted()).isEnabled(),
- "Cannot create inverted index for raw index column: %s", columnName);
+ if (originalConfig.getConfig(StandardIndexes.inverted()).isEnabled()) {
+ Preconditions.checkState(dictEnabledColumn,
+ "Cannot create inverted index for raw index column: %s",
columnName);
+ }
IndexType<ForwardIndexConfig, ?, ForwardIndexCreator> forwardIdx =
StandardIndexes.forward();
boolean forwardIndexDisabled =
!originalConfig.getConfig(forwardIdx).isEnabled();
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
index 0bc7845b8b..494de26a15 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
@@ -224,6 +224,10 @@ public class DictionaryIndexType
boolean optimizeDictionaryForMetrics, double
noDictionarySizeRatioThreshold,
FieldSpec fieldSpec, FieldIndexConfigs fieldIndexConfigs, int
cardinality,
int totalNumberOfEntries) {
+ // For an inverted index dictionary is required
+ if (fieldIndexConfigs.getConfig(StandardIndexes.inverted()).isEnabled()) {
+ return true;
+ }
if (optimizeDictionary) {
// Do not create dictionaries for json or text index columns as they are
high-cardinality values almost always
if ((fieldIndexConfigs.getConfig(StandardIndexes.json()).isEnabled() ||
fieldIndexConfigs.getConfig(
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
index 1df3e70364..333d4ffbbe 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandlerTest.java
@@ -26,6 +26,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -2377,6 +2378,30 @@ public class ForwardIndexHandlerTest {
Assert.assertEquals(result, true);
}
+ @Test
+ public void testInvertedIndexWithDictionaryHeuristics() {
+ FieldSpec fieldSpec = new MetricFieldSpec();
+ fieldSpec.setName("test");
+ fieldSpec.setDataType(FieldSpec.DataType.STRING);
+ IndexType index1 = Mockito.mock(IndexType.class);
+ Mockito.when(index1.getId()).thenReturn("index1");
+ IndexConfig indexConf = new IndexConfig(true);
+ FieldIndexConfigs fieldIndexConfigs = new
FieldIndexConfigs.Builder().add(index1, indexConf).build();
+ Map<String, FieldIndexConfigs> indexConfigs = new HashMap<>();
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(_tableConfig,
_schema);
+ config.setOutDir(INDEX_DIR.getPath());
+ config.setTableName(TABLE_NAME);
+ config.setSegmentName(SEGMENT_NAME);
+ config.setIndexOn(StandardIndexes.inverted(), IndexConfig.ENABLED,
_invertedIndexColumns);
+ config.setOptimizeDictionary(true);
+ config.setOptimizeDictionaryForMetrics(true);
+ indexConfigs.put("Column1", fieldIndexConfigs);
+ FieldIndexConfigs fieldIndexConfig =
config.getIndexConfigsByColName().get("Column1");
+ boolean result =
DictionaryIndexType.ignoreDictionaryOverride(config.isOptimizeDictionary(),
+ config.isOptimizeDictionaryForMetrics(), 2, fieldSpec,
fieldIndexConfigs, 5, 20);
+ Assert.assertEquals(result, true);
+ }
+
private void validateIndexesForForwardIndexDisabledColumns(String columnName)
throws IOException, ConfigurationException {
// Setup
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
