This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a8c3107144 Allow setting ForwardIndexConfig default settings via
cluster config (#14773)
a8c3107144 is described below
commit a8c310714481b25411c42786b5cfd0bbe1b4025a
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Thu Jan 9 01:03:07 2025 -0800
Allow setting ForwardIndexConfig default settings via cluster config
(#14773)
---
.../pinot/common/utils/ServiceStartableUtils.java | 39 +++++++++--
.../fwd/MultiValueFixedByteRawIndexCreator.java | 4 +-
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 6 +-
.../fwd/SingleValueFixedByteRawIndexCreator.java | 4 +-
.../fwd/SingleValueVarByteRawIndexCreator.java | 4 +-
.../segment/index/forward/ForwardIndexType.java | 6 +-
.../index/forward/ForwardIndexTypeTest.java | 19 +++---
.../index/loader/IndexLoadingConfigTest.java | 12 ++--
.../segment/spi/index/ForwardIndexConfig.java | 78 ++++++++++++++++------
.../spi/index/startree/AggregationSpec.java | 6 +-
.../segment/spi/index/ForwardIndexConfigTest.java | 8 +--
.../apache/pinot/spi/utils/CommonConstants.java | 9 +++
.../converter/DictionaryToRawIndexConverter.java | 4 +-
13 files changed, 137 insertions(+), 62 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/ServiceStartableUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/ServiceStartableUtils.java
index 45a791bc9a..cc14ad5454 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/ServiceStartableUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/ServiceStartableUtils.java
@@ -24,14 +24,13 @@ import java.util.concurrent.TimeUnit;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer;
import org.apache.helix.zookeeper.impl.client.ZkClient;
+import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.services.ServiceRole;
import org.apache.pinot.spi.utils.CommonConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.spi.utils.CommonConstants.CONFIG_OF_TIMEZONE;
-
public class ServiceStartableUtils {
private ServiceStartableUtils() {
@@ -44,7 +43,10 @@ public class ServiceStartableUtils {
protected static String _timeZone;
/**
- * Applies the ZK cluster config to the given instance config if it does not
already exist.
+ * Applies the ZK cluster config to:
+ * - The given instance config if it does not already exist.
+ * - Set the timezone.
+ * - Initialize the default values in {@link ForwardIndexConfig}.
*
* In the ZK cluster config:
* - pinot.all.* will be replaced to role specific config, e.g.
pinot.controller.* for controllers
@@ -70,7 +72,8 @@ public class ServiceStartableUtils {
zkClient.readData(String.format(CLUSTER_CONFIG_ZK_PATH_TEMPLATE,
clusterName, clusterName), true);
if (clusterConfigZNRecord == null) {
LOGGER.warn("Failed to find cluster config for cluster: {}, skipping
applying cluster config", clusterName);
- setupTimezone(instanceConfig);
+ setTimezone(instanceConfig);
+ initForwardIndexConfig(instanceConfig);
return;
}
@@ -92,7 +95,8 @@ public class ServiceStartableUtils {
} finally {
zkClient.close();
}
- setupTimezone(instanceConfig);
+ setTimezone(instanceConfig);
+ initForwardIndexConfig(instanceConfig);
}
private static void addConfigIfNotExists(PinotConfiguration instanceConfig,
String key, String value) {
@@ -101,10 +105,31 @@ public class ServiceStartableUtils {
}
}
- private static void setupTimezone(PinotConfiguration instanceConfig) {
+ private static void setTimezone(PinotConfiguration instanceConfig) {
TimeZone localTimezone = TimeZone.getDefault();
- _timeZone = instanceConfig.getProperty(CONFIG_OF_TIMEZONE,
localTimezone.getID());
+ _timeZone = instanceConfig.getProperty(CommonConstants.CONFIG_OF_TIMEZONE,
localTimezone.getID());
System.setProperty("user.timezone", _timeZone);
LOGGER.info("Timezone: {}", _timeZone);
}
+
+ private static void initForwardIndexConfig(PinotConfiguration
instanceConfig) {
+ String defaultRawIndexWriterVersion =
+
instanceConfig.getProperty(CommonConstants.ForwardIndexConfigs.CONFIG_OF_DEFAULT_RAW_INDEX_WRITER_VERSION);
+ if (defaultRawIndexWriterVersion != null) {
+ LOGGER.info("Setting forward index default raw index writer version to:
{}", defaultRawIndexWriterVersion);
+
ForwardIndexConfig.setDefaultRawIndexWriterVersion(Integer.parseInt(defaultRawIndexWriterVersion));
+ }
+ String defaultTargetMaxChunkSize =
+
instanceConfig.getProperty(CommonConstants.ForwardIndexConfigs.CONFIG_OF_DEFAULT_TARGET_MAX_CHUNK_SIZE);
+ if (defaultTargetMaxChunkSize != null) {
+ LOGGER.info("Setting forward index default target max chunk size to:
{}", defaultTargetMaxChunkSize);
+
ForwardIndexConfig.setDefaultTargetMaxChunkSize(defaultTargetMaxChunkSize);
+ }
+ String defaultTargetDocsPerChunk =
+
instanceConfig.getProperty(CommonConstants.ForwardIndexConfigs.CONFIG_OF_DEFAULT_TARGET_DOCS_PER_CHUNK);
+ if (defaultTargetDocsPerChunk != null) {
+ LOGGER.info("Setting forward index default target docs per chunk to:
{}", defaultTargetDocsPerChunk);
+
ForwardIndexConfig.setDefaultTargetDocsPerChunk(Integer.parseInt(defaultTargetDocsPerChunk));
+ }
+ }
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
index b8a6bd6daa..346fd883fe 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
@@ -66,8 +66,8 @@ public class MultiValueFixedByteRawIndexCreator implements
ForwardIndexCreator {
DataType valueType, int maxNumberOfMultiValueElements, boolean
deriveNumDocsPerChunk, int writerVersion)
throws IOException {
this(indexFile, compressionType, totalDocs, valueType,
maxNumberOfMultiValueElements, deriveNumDocsPerChunk,
- writerVersion, ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES,
- ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ writerVersion, ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes(),
+ ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
public MultiValueFixedByteRawIndexCreator(File indexFile,
ChunkCompressionType compressionType, int totalDocs,
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
index a31f1031b9..21cda225d0 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
@@ -54,9 +54,9 @@ public class MultiValueVarByteRawIndexCreator implements
ForwardIndexCreator {
public MultiValueVarByteRawIndexCreator(File baseIndexDir,
ChunkCompressionType compressionType, String column,
int totalDocs, DataType valueType, int maxRowLengthInBytes, int
maxNumberOfElements)
throws IOException {
- this(baseIndexDir, compressionType, column, totalDocs, valueType,
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
- maxRowLengthInBytes, maxNumberOfElements,
ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES,
- ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ this(baseIndexDir, compressionType, column, totalDocs, valueType,
ForwardIndexConfig.getDefaultRawWriterVersion(),
+ maxRowLengthInBytes, maxNumberOfElements,
ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes(),
+ ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
/**
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
index c509650ee2..453519c8a6 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
@@ -49,8 +49,8 @@ public class SingleValueFixedByteRawIndexCreator implements
ForwardIndexCreator
public SingleValueFixedByteRawIndexCreator(File baseIndexDir,
ChunkCompressionType compressionType, String column,
int totalDocs, DataType valueType)
throws IOException {
- this(baseIndexDir, compressionType, column, totalDocs, valueType,
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
- ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ this(baseIndexDir, compressionType, column, totalDocs, valueType,
ForwardIndexConfig.getDefaultRawWriterVersion(),
+ ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
/**
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
index 5b5a1ff0e3..40a803b0a1 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
@@ -54,8 +54,8 @@ public class SingleValueVarByteRawIndexCreator implements
ForwardIndexCreator {
int totalDocs, DataType valueType, int maxLength)
throws IOException {
this(baseIndexDir, compressionType, column, totalDocs, valueType,
maxLength, false,
- ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES,
- ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ ForwardIndexConfig.getDefaultRawWriterVersion(),
ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes(),
+ ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
/**
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
index 03ed28b2f0..c9b49bbc36 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
@@ -87,7 +87,7 @@ public class ForwardIndexType extends
AbstractIndexType<ForwardIndexConfig, Forw
@Override
public ForwardIndexConfig getDefaultConfig() {
- return ForwardIndexConfig.DEFAULT;
+ return ForwardIndexConfig.getDefault();
}
@Override
@@ -109,10 +109,10 @@ public class ForwardIndexType extends
AbstractIndexType<ForwardIndexConfig, Forw
for (FieldConfig fieldConfig : fieldConfigs) {
Map<String, String> properties = fieldConfig.getProperties();
if (properties != null && isDisabled(properties)) {
- fwdConfig.put(fieldConfig.getName(), ForwardIndexConfig.DISABLED);
+ fwdConfig.put(fieldConfig.getName(),
ForwardIndexConfig.getDisabled());
} else {
ForwardIndexConfig config =
createConfigFromFieldConfig(fieldConfig);
- if (!config.equals(ForwardIndexConfig.DEFAULT)) {
+ if (!config.equals(ForwardIndexConfig.getDefault())) {
fwdConfig.put(fieldConfig.getName(), config);
}
// It is important to do not explicitly add the default value here
in order to avoid exclusive problems with
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
index 66bd92b2e2..12f53908be 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
@@ -92,7 +92,7 @@ public class ForwardIndexTypeTest {
JsonUtils.stringToObject("[]", _fieldConfigListTypeRef)
);
- assertEquals(ForwardIndexConfig.DEFAULT);
+ assertEquals(ForwardIndexConfig.getDefault());
}
@Test
@@ -108,7 +108,7 @@ public class ForwardIndexTypeTest {
+ " }]", _fieldConfigListTypeRef)
);
- assertEquals(ForwardIndexConfig.DISABLED);
+ assertEquals(ForwardIndexConfig.getDisabled());
}
@Test
@@ -120,7 +120,7 @@ public class ForwardIndexTypeTest {
+ " }"
);
- assertEquals(ForwardIndexConfig.DEFAULT);
+ assertEquals(ForwardIndexConfig.getDefault());
}
@Test
@@ -177,7 +177,7 @@ public class ForwardIndexTypeTest {
+ " \"encodingType\": \"DICTIONARY\"\n"
+ " }"
);
- assertEquals(ForwardIndexConfig.DEFAULT);
+ assertEquals(ForwardIndexConfig.getDefault());
}
@Test
@@ -204,7 +204,7 @@ public class ForwardIndexTypeTest {
+ " }"
);
- assertEquals(ForwardIndexConfig.DEFAULT);
+ assertEquals(ForwardIndexConfig.getDefault());
}
@Test(dataProvider = "allCompressionCodec", dataProviderClass =
ForwardIndexTypeTest.class)
@@ -227,7 +227,7 @@ public class ForwardIndexTypeTest {
.withCompressionType(expectedChunkCompression)
.withDictIdCompressionType(expectedDictCompression)
.withDeriveNumDocsPerChunk(false)
-
.withRawIndexWriterVersion(ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION)
+
.withRawIndexWriterVersion(ForwardIndexConfig.getDefaultRawWriterVersion())
.build()
);
}
@@ -248,7 +248,7 @@ public class ForwardIndexTypeTest {
assertEquals(new ForwardIndexConfig.Builder()
.withCompressionType(null)
.withDeriveNumDocsPerChunk(true)
-
.withRawIndexWriterVersion(ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION)
+
.withRawIndexWriterVersion(ForwardIndexConfig.getDefaultRawWriterVersion())
.build());
}
@@ -284,7 +284,8 @@ public class ForwardIndexTypeTest {
+ " }\n"
+ " }"
);
- assertEquals(ForwardIndexConfig.DISABLED);
+
+ assertEquals(ForwardIndexConfig.getDisabled());
}
@Test
@@ -297,7 +298,7 @@ public class ForwardIndexTypeTest {
+ " }"
);
- assertEquals(ForwardIndexConfig.DEFAULT);
+ assertEquals(ForwardIndexConfig.getDefault());
}
@Test
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfigTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfigTest.java
index a717973a64..18ee15285a 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfigTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfigTest.java
@@ -183,9 +183,9 @@ public class IndexLoadingConfigTest {
assertTrue(forwardIndexConfig.isEnabled());
assertNull(forwardIndexConfig.getCompressionCodec());
assertFalse(forwardIndexConfig.isDeriveNumDocsPerChunk());
- assertEquals(forwardIndexConfig.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION);
- assertEquals(forwardIndexConfig.getTargetMaxChunkSize(),
ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE);
- assertEquals(forwardIndexConfig.getTargetDocsPerChunk(),
ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ assertEquals(forwardIndexConfig.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion());
+ assertEquals(forwardIndexConfig.getTargetMaxChunkSize(),
ForwardIndexConfig.getDefaultTargetMaxChunkSize());
+ assertEquals(forwardIndexConfig.getTargetDocsPerChunk(),
ForwardIndexConfig.getDefaultTargetDocsPerChunk());
// Check custom settings
//@formatter:off
@@ -242,8 +242,8 @@ public class IndexLoadingConfigTest {
assertFalse(forwardIndexConfig.isEnabled());
assertNull(forwardIndexConfig.getCompressionCodec());
assertFalse(forwardIndexConfig.isDeriveNumDocsPerChunk());
- assertEquals(forwardIndexConfig.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION);
- assertEquals(forwardIndexConfig.getTargetMaxChunkSize(),
ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE);
- assertEquals(forwardIndexConfig.getTargetDocsPerChunk(),
ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ assertEquals(forwardIndexConfig.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion());
+ assertEquals(forwardIndexConfig.getTargetMaxChunkSize(),
ForwardIndexConfig.getDefaultTargetMaxChunkSize());
+ assertEquals(forwardIndexConfig.getTargetDocsPerChunk(),
ForwardIndexConfig.getDefaultTargetDocsPerChunk());
}
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
index 89b5a95d4f..fe2cfbbd2e 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.pinot.segment.spi.index;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -35,14 +34,56 @@ import org.apache.pinot.spi.utils.DataSizeUtils;
public class ForwardIndexConfig extends IndexConfig {
+ @Deprecated
public static final int DEFAULT_RAW_WRITER_VERSION = 2;
- public static final int DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES = 1024 * 1024;
// 1MB
- public static final String DEFAULT_TARGET_MAX_CHUNK_SIZE =
- DataSizeUtils.fromBytes(DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES);
+ @Deprecated
+ public static final String DEFAULT_TARGET_MAX_CHUNK_SIZE = "1MB";
+ @Deprecated
+ public static final int DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES = 1024 * 1024;
+ @Deprecated
public static final int DEFAULT_TARGET_DOCS_PER_CHUNK = 1000;
- public static final ForwardIndexConfig DISABLED =
- new ForwardIndexConfig(true, null, null, null, null, null, null, null);
- public static final ForwardIndexConfig DEFAULT = new Builder().build();
+
+ private static int _defaultRawIndexWriterVersion = 2;
+ private static String _defaultTargetMaxChunkSize = "1MB";
+ private static int _defaultTargetMaxChunkSizeBytes = 1024 * 1024;
+ private static int _defaultTargetDocsPerChunk = 1000;
+
+ public static int getDefaultRawWriterVersion() {
+ return _defaultRawIndexWriterVersion;
+ }
+
+ public static void setDefaultRawIndexWriterVersion(int
defaultRawIndexWriterVersion) {
+ _defaultRawIndexWriterVersion = defaultRawIndexWriterVersion;
+ }
+
+ public static String getDefaultTargetMaxChunkSize() {
+ return _defaultTargetMaxChunkSize;
+ }
+
+ public static int getDefaultTargetMaxChunkSizeBytes() {
+ return _defaultTargetMaxChunkSizeBytes;
+ }
+
+ public static void setDefaultTargetMaxChunkSize(String
defaultTargetMaxChunkSize) {
+ _defaultTargetMaxChunkSize = defaultTargetMaxChunkSize;
+ _defaultTargetMaxChunkSizeBytes = (int)
DataSizeUtils.toBytes(defaultTargetMaxChunkSize);
+ }
+
+ public static int getDefaultTargetDocsPerChunk() {
+ return _defaultTargetDocsPerChunk;
+ }
+
+ public static void setDefaultTargetDocsPerChunk(int
defaultTargetDocsPerChunk) {
+ _defaultTargetDocsPerChunk = defaultTargetDocsPerChunk;
+ }
+
+ public static ForwardIndexConfig getDefault() {
+ return new Builder().build();
+ }
+
+ public static ForwardIndexConfig getDisabled() {
+ return new ForwardIndexConfig(true, null, null, null, null, null, null,
null);
+ }
@Nullable
private final CompressionCodec _compressionCodec;
@@ -61,15 +102,14 @@ public class ForwardIndexConfig extends IndexConfig {
@Nullable Boolean deriveNumDocsPerChunk, @Nullable Integer
rawIndexWriterVersion,
@Nullable String targetMaxChunkSize, @Nullable Integer
targetDocsPerChunk) {
super(disabled);
- _deriveNumDocsPerChunk = Boolean.TRUE.equals(deriveNumDocsPerChunk);
- _rawIndexWriterVersion = rawIndexWriterVersion == null ?
DEFAULT_RAW_WRITER_VERSION : rawIndexWriterVersion;
_compressionCodec = compressionCodec;
+ _deriveNumDocsPerChunk = Boolean.TRUE.equals(deriveNumDocsPerChunk);
- _targetMaxChunkSizeBytes = targetMaxChunkSize == null ?
DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES
- : (int) DataSizeUtils.toBytes(targetMaxChunkSize);
- _targetMaxChunkSize =
- targetMaxChunkSize == null ? DEFAULT_TARGET_MAX_CHUNK_SIZE :
targetMaxChunkSize;
- _targetDocsPerChunk = targetDocsPerChunk == null ?
DEFAULT_TARGET_DOCS_PER_CHUNK : targetDocsPerChunk;
+ _rawIndexWriterVersion = rawIndexWriterVersion == null ?
_defaultRawIndexWriterVersion : rawIndexWriterVersion;
+ _targetMaxChunkSize = targetMaxChunkSize == null ?
_defaultTargetMaxChunkSize : targetMaxChunkSize;
+ _targetMaxChunkSizeBytes =
+ targetMaxChunkSize == null ? _defaultTargetMaxChunkSizeBytes : (int)
DataSizeUtils.toBytes(targetMaxChunkSize);
+ _targetDocsPerChunk = targetDocsPerChunk == null ?
_defaultTargetDocsPerChunk : targetDocsPerChunk;
if (compressionCodec != null) {
switch (compressionCodec) {
@@ -115,10 +155,10 @@ public class ForwardIndexConfig extends IndexConfig {
@Deprecated @JsonProperty("dictIdCompressionType") @Nullable
DictIdCompressionType dictIdCompressionType,
@JsonProperty("deriveNumDocsPerChunk") @Nullable Boolean
deriveNumDocsPerChunk,
@JsonProperty("rawIndexWriterVersion") @Nullable Integer
rawIndexWriterVersion,
- @JsonProperty("targetMaxChunkSize") @Nullable String
targetMaxChunkSizeBytes,
+ @JsonProperty("targetMaxChunkSize") @Nullable String targetMaxChunkSize,
@JsonProperty("targetDocsPerChunk") @Nullable Integer
targetDocsPerChunk) {
this(disabled, getActualCompressionCodec(compressionCodec,
chunkCompressionType, dictIdCompressionType),
- deriveNumDocsPerChunk, rawIndexWriterVersion, targetMaxChunkSizeBytes,
targetDocsPerChunk);
+ deriveNumDocsPerChunk, rawIndexWriterVersion, targetMaxChunkSize,
targetDocsPerChunk);
}
public static CompressionCodec getActualCompressionCodec(@Nullable
CompressionCodec compressionCodec,
@@ -219,9 +259,9 @@ public class ForwardIndexConfig extends IndexConfig {
@Nullable
private CompressionCodec _compressionCodec;
private boolean _deriveNumDocsPerChunk = false;
- private int _rawIndexWriterVersion = DEFAULT_RAW_WRITER_VERSION;
- private String _targetMaxChunkSize;
- private int _targetDocsPerChunk = DEFAULT_TARGET_DOCS_PER_CHUNK;
+ private int _rawIndexWriterVersion = _defaultRawIndexWriterVersion;
+ private String _targetMaxChunkSize = _defaultTargetMaxChunkSize;
+ private int _targetDocsPerChunk = _defaultTargetDocsPerChunk;
public Builder() {
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationSpec.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationSpec.java
index a4a762fb88..4473261e4d 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationSpec.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/startree/AggregationSpec.java
@@ -48,13 +48,13 @@ public class AggregationSpec {
public AggregationSpec(@Nullable CompressionCodec compressionCodec,
@Nullable Boolean deriveNumDocsPerChunk,
@Nullable Integer indexVersion, @Nullable Integer
targetMaxChunkSizeBytes, @Nullable Integer targetDocsPerChunk,
@Nullable Map<String, Object> functionParameters) {
- _indexVersion = indexVersion != null ? indexVersion :
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION;
+ _indexVersion = indexVersion != null ? indexVersion :
ForwardIndexConfig.getDefaultRawWriterVersion();
_compressionCodec = compressionCodec != null ? compressionCodec :
DEFAULT_COMPRESSION_CODEC;
_deriveNumDocsPerChunk = deriveNumDocsPerChunk != null ?
deriveNumDocsPerChunk : false;
_targetMaxChunkSizeBytes = targetMaxChunkSizeBytes != null ?
targetMaxChunkSizeBytes
- : ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES;
+ : ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes();
_targetDocsPerChunk =
- targetDocsPerChunk != null ? targetDocsPerChunk :
ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK;
+ targetDocsPerChunk != null ? targetDocsPerChunk :
ForwardIndexConfig.getDefaultTargetDocsPerChunk();
_functionParameters = functionParameters == null ? Map.of() :
functionParameters;
}
diff --git
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/ForwardIndexConfigTest.java
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/ForwardIndexConfigTest.java
index 58adf57014..33b1f61f20 100644
---
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/ForwardIndexConfigTest.java
+++
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/ForwardIndexConfigTest.java
@@ -37,7 +37,7 @@ public class ForwardIndexConfigTest {
assertFalse(config.isDisabled(), "Unexpected disabled");
assertNull(config.getChunkCompressionType(), "Unexpected
chunkCompressionType");
assertFalse(config.isDeriveNumDocsPerChunk(), "Unexpected
deriveNumDocsPerChunk");
- assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
+ assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion(),
"Unexpected rawIndexWriterVersion");
}
@@ -50,7 +50,7 @@ public class ForwardIndexConfigTest {
assertFalse(config.isDisabled(), "Unexpected disabled");
assertNull(config.getChunkCompressionType(), "Unexpected
chunkCompressionType");
assertFalse(config.isDeriveNumDocsPerChunk(), "Unexpected
deriveNumDocsPerChunk");
- assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
+ assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion(),
"Unexpected rawIndexWriterVersion");
}
@@ -63,7 +63,7 @@ public class ForwardIndexConfigTest {
assertFalse(config.isDisabled(), "Unexpected disabled");
assertNull(config.getChunkCompressionType(), "Unexpected
chunkCompressionType");
assertFalse(config.isDeriveNumDocsPerChunk(), "Unexpected
deriveNumDocsPerChunk");
- assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
+ assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion(),
"Unexpected rawIndexWriterVersion");
}
@@ -76,7 +76,7 @@ public class ForwardIndexConfigTest {
assertTrue(config.isDisabled(), "Unexpected disabled");
assertNull(config.getChunkCompressionType(), "Unexpected
chunkCompressionType");
assertFalse(config.isDeriveNumDocsPerChunk(), "Unexpected
deriveNumDocsPerChunk");
- assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
+ assertEquals(config.getRawIndexWriterVersion(),
ForwardIndexConfig.getDefaultRawWriterVersion(),
"Unexpected rawIndexWriterVersion");
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index f425e36725..0156c49f6d 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1347,4 +1347,13 @@ public class CommonConstants {
public static final String RESPONSE_STORE_CLEANER_INITIAL_DELAY =
"controller.cluster.response.store.cleaner.initialDelay";
}
+
+ public static class ForwardIndexConfigs {
+ public static final String CONFIG_OF_DEFAULT_RAW_INDEX_WRITER_VERSION =
+ "pinot.forward.index.default.raw.index.writer.version";
+ public static final String CONFIG_OF_DEFAULT_TARGET_MAX_CHUNK_SIZE =
+ "pinot.forward.index.default.target.max.chunk.size";
+ public static final String CONFIG_OF_DEFAULT_TARGET_DOCS_PER_CHUNK =
+ "pinot.forward.index.default.target.docs.per.chunk";
+ }
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
index 65660b00ba..065bd27d85 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
@@ -318,8 +318,8 @@ public class DictionaryToRawIndexConverter {
try (ForwardIndexCreator rawIndexCreator =
ForwardIndexCreatorFactory.getRawIndexCreatorForSVColumn(newSegment,
compressionType, column, storedType, numDocs, lengthOfLongestEntry,
false,
- ForwardIndexConfig.DEFAULT_RAW_WRITER_VERSION,
ForwardIndexConfig.DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES,
- ForwardIndexConfig.DEFAULT_TARGET_DOCS_PER_CHUNK);
+ ForwardIndexConfig.getDefaultRawWriterVersion(),
ForwardIndexConfig.getDefaultTargetMaxChunkSizeBytes(),
+ ForwardIndexConfig.getDefaultTargetDocsPerChunk());
ForwardIndexReaderContext readerContext =
forwardIndexReader.createContext()) {
switch (storedType) {
case INT:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
