This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5df2ffbdbc7 Add global pinot.md5.disabled switch and enforce MD5
guards (#17800)
5df2ffbdbc7 is described below
commit 5df2ffbdbc7701f72a72488a7ef7c911ff2895a9
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Mar 3 15:19:15 2026 -0800
Add global pinot.md5.disabled switch and enforce MD5 guards (#17800)
---
.../broker/broker/helix/BaseBrokerStarter.java | 3 +
.../pinot/controller/BaseControllerStarter.java | 3 +
.../org/apache/pinot/minion/BaseMinionStarter.java | 3 +
.../pinot/plugin/filesystem/ADLSGen2PinotFS.java | 13 +++-
.../filesystem/test/ADLSGen2PinotFSTest.java | 29 ++++++++
.../apache/pinot/plugin/filesystem/S3Config.java | 6 ++
.../pinot/plugin/filesystem/S3ConfigTest.java | 13 ++++
.../pinot/segment/local/utils/HashUtils.java | 6 ++
.../segment/local/utils/TableConfigUtils.java | 15 ++++-
.../pinot/segment/local/utils/HashUtilsTest.java | 16 +++++
.../segment/local/utils/TableConfigUtilsTest.java | 77 ++++++++++++++++++++++
.../server/starter/helix/BaseServerStarter.java | 3 +
.../apache/pinot/spi/utils/CommonConstants.java | 1 +
.../org/apache/pinot/spi/utils/PinotMd5Mode.java | 46 +++++++++++++
.../apache/pinot/spi/utils/PinotMd5ModeTest.java | 60 +++++++++++++++++
15 files changed, 291 insertions(+), 3 deletions(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
index 02a1e9bca59..52dfca19fc7 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/BaseBrokerStarter.java
@@ -116,6 +116,7 @@ import org.apache.pinot.spi.utils.CommonConstants.Helix;
import org.apache.pinot.spi.utils.CommonConstants.MultiStageQueryRunner;
import org.apache.pinot.spi.utils.InstanceTypeUtils;
import org.apache.pinot.spi.utils.NetUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.sql.parsers.rewriter.QueryRewriterFactory;
import org.apache.pinot.tsdb.spi.PinotTimeSeriesConfiguration;
import org.slf4j.Logger;
@@ -190,6 +191,8 @@ public abstract class BaseBrokerStarter implements
ServiceStartable {
PinotInsecureMode.setPinotInInsecureMode(
_brokerConf.getProperty(CommonConstants.CONFIG_OF_PINOT_INSECURE_MODE,
false));
+
PinotMd5Mode.setPinotMd5Disabled(_brokerConf.getProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
+ PinotMd5Mode.isPinotMd5Disabled()));
if (_brokerConf.getProperty(MultiStageQueryRunner.KEY_OF_QUERY_RUNNER_PORT,
MultiStageQueryRunner.DEFAULT_QUERY_RUNNER_PORT) == 0) {
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index d8373ca4486..31ed430eeec 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -161,6 +161,7 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Helix;
import org.apache.pinot.spi.utils.InstanceTypeUtils;
import org.apache.pinot.spi.utils.NetUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.sql.parsers.rewriter.QueryRewriterFactory;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.slf4j.Logger;
@@ -251,6 +252,8 @@ public abstract class BaseControllerStarter implements
ServiceStartable {
applyCustomConfigs(_config);
PinotInsecureMode.setPinotInInsecureMode(_config.getProperty(CommonConstants.CONFIG_OF_PINOT_INSECURE_MODE,
false));
+
PinotMd5Mode.setPinotMd5Disabled(_config.getProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
+ PinotMd5Mode.isPinotMd5Disabled()));
setupHelixSystemProperties();
IdealStateGroupCommit.setMinNumCharsInISToTurnOnCompression(_config.getMinNumCharsInISToTurnOnCompression());
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
index fbcbff886c8..9941edacee9 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
@@ -77,6 +77,7 @@ import org.apache.pinot.spi.services.ServiceStartable;
import org.apache.pinot.spi.tasks.MinionTaskObserverStorageManager;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.InstanceTypeUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.sql.parsers.rewriter.QueryRewriterFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,6 +114,8 @@ public abstract class BaseMinionStarter implements
ServiceStartable {
applyCustomConfigs(_config);
PinotInsecureMode.setPinotInInsecureMode(_config.getProperty(CommonConstants.CONFIG_OF_PINOT_INSECURE_MODE,
false));
+
PinotMd5Mode.setPinotMd5Disabled(_config.getProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
+ PinotMd5Mode.isPinotMd5Disabled()));
setupHelixSystemProperties();
_hostname = _config.getHostName();
diff --git
a/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/ADLSGen2PinotFS.java
b/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/ADLSGen2PinotFS.java
index 0f294f9103e..e3b3cc01baa 100644
---
a/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/ADLSGen2PinotFS.java
+++
b/pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/ADLSGen2PinotFS.java
@@ -64,6 +64,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.BasePinotFS;
import org.apache.pinot.spi.filesystem.FileMetadata;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,7 +125,14 @@ public class ADLSGen2PinotFS extends BasePinotFS {
@Override
public void init(PinotConfiguration config) {
- _enableChecksum = config.getProperty(ENABLE_CHECKSUM, false);
+ boolean checksumEnabled = config.getProperty(ENABLE_CHECKSUM, false);
+ if (checksumEnabled && PinotMd5Mode.isPinotMd5Disabled()) {
+ throw new IllegalStateException(String.format(
+ "ADLS checksum requires MD5, but MD5 is disabled via '%s=true'. Set
'%s=false' or '%s=false'.",
+ CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
+ ENABLE_CHECKSUM));
+ }
+ _enableChecksum = checksumEnabled;
// Azure storage account name
String accountName = config.getProperty(ACCOUNT_NAME);
@@ -539,7 +548,7 @@ public class ADLSGen2PinotFS extends BasePinotFS {
public void copyFromLocalFile(File srcFile, URI dstUri)
throws Exception {
LOGGER.debug("copyFromLocalFile is called with srcFile='{}', dstUri='{}'",
srcFile, dstUri);
- byte[] contentMd5 = computeContentMd5(srcFile);
+ byte[] contentMd5 = _enableChecksum ? computeContentMd5(srcFile) : null;
try (InputStream fileInputStream = new FileInputStream(srcFile)) {
copyInputStreamToDst(fileInputStream, dstUri, contentMd5);
}
diff --git
a/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/ADLSGen2PinotFSTest.java
b/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/ADLSGen2PinotFSTest.java
index 4202ae7fdfd..c7823006520 100644
---
a/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/ADLSGen2PinotFSTest.java
+++
b/pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/ADLSGen2PinotFSTest.java
@@ -47,6 +47,7 @@ import org.apache.pinot.plugin.filesystem.ADLSGen2PinotFS;
import org.apache.pinot.plugin.filesystem.AzurePinotFSUtil;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.FileMetadata;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.AfterMethod;
@@ -141,6 +142,34 @@ public class ADLSGen2PinotFSTest {
assertTrue(sasTokenFS != null);
}
+ @Test
+ public void testChecksumEnabledWithMd5DisabledFails() {
+ PinotConfiguration pinotConfiguration = new PinotConfiguration();
+ pinotConfiguration.setProperty("authenticationType", "SAS_TOKEN");
+ pinotConfiguration.setProperty("sasToken",
"sp=rwdl&se=2025-12-31T23:59:59Z&sv=2022-11-02&sr=c&sig=test");
+ pinotConfiguration.setProperty("accountName", "testaccount");
+ pinotConfiguration.setProperty("fileSystemName", "testcontainer");
+ pinotConfiguration.setProperty("enableChecksum", "true");
+
+ ADLSGen2PinotFS adlsGen2PinotFs = new ADLSGen2PinotFS() {
+ @Override
+ public DataLakeFileSystemClient
getOrCreateClientWithFileSystem(DataLakeServiceClient serviceClient,
+ String fileSystemName) {
+ return _mockFileSystemClient;
+ }
+ };
+
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ IllegalStateException exception =
+ expectThrows(IllegalStateException.class, () ->
adlsGen2PinotFs.init(pinotConfiguration));
+ assertTrue(exception.getMessage().contains("pinot.md5.disabled"));
+ assertTrue(exception.getMessage().contains("enableChecksum"));
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
@Test(expectedExceptions = NullPointerException.class)
public void testSasTokenMissingToken() {
PinotConfiguration pinotConfiguration = new PinotConfiguration();
diff --git
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3Config.java
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3Config.java
index a38db4c9072..8666d888934 100644
---
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3Config.java
+++
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3Config.java
@@ -25,7 +25,9 @@ import java.time.Duration;
import java.util.UUID;
import javax.annotation.Nullable;
import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.DataSizeUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.spi.utils.TimeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -126,6 +128,10 @@ public class S3Config {
_responseChecksumValidationWhenRequired =
ResponseChecksumValidation.fromValue(
pinotConfig.getProperty(RESPONSE_CHECKSUM_VALIDATION,
ResponseChecksumValidation.WHEN_REQUIRED.name()));
_useLegacyMd5Plugin =
Boolean.parseBoolean(pinotConfig.getProperty(USE_LEGACY_MD5_PLUGIN, "false"));
+ if (_useLegacyMd5Plugin && PinotMd5Mode.isPinotMd5Disabled()) {
+ throw new IllegalStateException(String.format("S3 config '%s=true' is
not allowed when '%s=true'",
+ USE_LEGACY_MD5_PLUGIN,
CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ }
_storageClass = pinotConfig.getProperty(STORAGE_CLASS);
if (_storageClass != null) {
diff --git
a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3ConfigTest.java
b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3ConfigTest.java
index 0f4528fd022..3eae446fa51 100644
---
a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3ConfigTest.java
+++
b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3ConfigTest.java
@@ -19,6 +19,7 @@
package org.apache.pinot.plugin.filesystem;
import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.testng.Assert;
import org.testng.annotations.Test;
import software.amazon.awssdk.services.s3.model.StorageClass;
@@ -75,4 +76,16 @@ public class S3ConfigTest {
S3Config cfg = new S3Config(pinotConfig);
Assert.assertTrue(cfg.useLegacyMd5Plugin());
}
+
+ @Test(expectedExceptions = IllegalStateException.class)
+ public void testLegacyMd5PluginWhenMd5Disabled() {
+ PinotConfiguration pinotConfig = new PinotConfiguration();
+ pinotConfig.setProperty("useLegacyMd5Plugin", "true");
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ new S3Config(pinotConfig);
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/HashUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/HashUtils.java
index 5070bbf9270..578280fb0d2 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/HashUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/HashUtils.java
@@ -28,6 +28,8 @@ import net.jpountz.xxhash.XXHashFactory;
import org.apache.pinot.spi.config.table.HashFunction;
import org.apache.pinot.spi.data.readers.PrimaryKey;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
public class HashUtils {
@@ -94,6 +96,10 @@ public class HashUtils {
case NONE:
return primaryKey;
case MD5:
+ if (PinotMd5Mode.isPinotMd5Disabled()) {
+ throw new IllegalStateException(String.format("Hash function MD5 is
disabled via '%s=true'",
+ CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ }
return new ByteArray(HashUtils.hashMD5(primaryKey.asBytes()));
case MURMUR3:
return new ByteArray(HashUtils.hashMurmur3(primaryKey.asBytes()));
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index 97623115c3c..a8453a1c3bf 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -62,6 +62,7 @@ import
org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.DedupConfig;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.config.table.HashFunction;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.MultiColumnTextIndexConfig;
import org.apache.pinot.spi.config.table.QuotaConfig;
@@ -104,6 +105,7 @@ import org.apache.pinot.spi.utils.DataSizeUtils;
import org.apache.pinot.spi.utils.Enablement;
import org.apache.pinot.spi.utils.IngestionConfigUtils;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.spi.utils.TimeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -777,7 +779,8 @@ public final class TableConfigUtils {
}
boolean isUpsertEnabled = tableConfig.getUpsertMode() !=
UpsertConfig.Mode.NONE;
- boolean isDedupEnabled = tableConfig.getDedupConfig() != null &&
tableConfig.getDedupConfig().isDedupEnabled();
+ DedupConfig dedupConfig = tableConfig.getDedupConfig();
+ boolean isDedupEnabled = dedupConfig != null &&
dedupConfig.isDedupEnabled();
// check both upsert and dedup are not enabled simultaneously
Preconditions.checkState(!(isUpsertEnabled && isDedupEnabled),
@@ -807,6 +810,16 @@ public final class TableConfigUtils {
// specifically for upsert
UpsertConfig upsertConfig = tableConfig.getUpsertConfig();
+ if (PinotMd5Mode.isPinotMd5Disabled()) {
+ if (isUpsertEnabled && upsertConfig != null &&
upsertConfig.getHashFunction() == HashFunction.MD5) {
+ throw new IllegalStateException(String.format(
+ "Upsert hash function MD5 is disabled via '%s=true'",
CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ }
+ if (isDedupEnabled && dedupConfig.getHashFunction() == HashFunction.MD5)
{
+ throw new IllegalStateException(String.format(
+ "Dedup hash function MD5 is disabled via '%s=true'",
CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ }
+ }
if (upsertConfig != null) {
// Currently, only one tier is allowed for upsert table, as the
committed segments can't be moved to other tiers.
Preconditions.checkState(tableConfig.getTierConfigsList() == null, "The
upsert table cannot have multi-tiers");
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/HashUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/HashUtilsTest.java
index 9f4649b162a..8c60ad6f374 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/HashUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/HashUtilsTest.java
@@ -19,8 +19,11 @@
package org.apache.pinot.segment.local.utils;
import java.util.UUID;
+import org.apache.pinot.spi.config.table.HashFunction;
import org.apache.pinot.spi.data.readers.PrimaryKey;
import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
@@ -61,6 +64,19 @@ public class HashUtilsTest {
}
}
+ @Test
+ public void testHashPrimaryKeyWithMd5Disabled() {
+ PrimaryKey primaryKey = new PrimaryKey(new Object[]{"hello world"});
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ IllegalStateException exception =
+ expectThrows(IllegalStateException.class, () ->
HashUtils.hashPrimaryKey(primaryKey, HashFunction.MD5));
+
assertTrue(exception.getMessage().contains(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
private void testHashUUID(UUID[] uuids) {
byte[] convertedBytes = HashUtils.hashUUID(new PrimaryKey(uuids));
// After hashing, each UUID should take 16 bytes.
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
index 1488bf81e51..0194c45633c 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java
@@ -38,6 +38,7 @@ import
org.apache.pinot.spi.config.table.ColumnPartitionConfig;
import org.apache.pinot.spi.config.table.DedupConfig;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.FieldConfig.CompressionCodec;
+import org.apache.pinot.spi.config.table.HashFunction;
import org.apache.pinot.spi.config.table.IndexingConfig;
import org.apache.pinot.spi.config.table.ReplicaGroupStrategyConfig;
import org.apache.pinot.spi.config.table.RoutingConfig;
@@ -79,6 +80,7 @@ import org.apache.pinot.spi.stream.StreamMetadataProvider;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.Enablement;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.mockito.Mockito;
import org.testng.annotations.DataProvider;
@@ -2025,6 +2027,81 @@ public class TableConfigUtilsTest {
}
}
+ @Test
+ public void testValidateUpsertConfigWithMd5Disabled() {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
+ .setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
+ UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
+ upsertConfig.setHashFunction(HashFunction.MD5);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(TABLE_NAME)
+ .setUpsertConfig(upsertConfig)
+ .setRoutingConfig(
+ new RoutingConfig(null, null,
RoutingConfig.STRICT_REPLICA_GROUP_INSTANCE_SELECTOR_TYPE, false))
+ .setStreamConfigs(getStreamConfigs())
+ .build();
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ IllegalStateException exception =
+ expectThrows(IllegalStateException.class, () ->
TableConfigUtils.validateUpsertAndDedupConfig(tableConfig,
+ schema));
+
assertTrue(exception.getMessage().contains(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
+ @Test
+ public void testValidateDedupConfigWithMd5Disabled() {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
+ .setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
+ DedupConfig dedupConfig = new DedupConfig();
+ dedupConfig.setHashFunction(HashFunction.MD5);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(TABLE_NAME)
+ .setDedupConfig(dedupConfig)
+ .setRoutingConfig(
+ new RoutingConfig(null, null,
RoutingConfig.STRICT_REPLICA_GROUP_INSTANCE_SELECTOR_TYPE, false))
+ .setStreamConfigs(getStreamConfigs())
+ .build();
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ IllegalStateException exception =
+ expectThrows(IllegalStateException.class, () ->
TableConfigUtils.validateUpsertAndDedupConfig(tableConfig,
+ schema));
+
assertTrue(exception.getMessage().contains(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED));
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
+ @Test
+ public void testValidateDedupConfigWithMd5DisabledAllowsUpsertModeNoneMd5() {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
+ .addSingleValueDimension("myCol", FieldSpec.DataType.STRING)
+ .setPrimaryKeyColumns(Lists.newArrayList("myCol"))
+ .build();
+ DedupConfig dedupConfig = new DedupConfig();
+ dedupConfig.setHashFunction(HashFunction.NONE);
+ UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.NONE);
+ upsertConfig.setHashFunction(HashFunction.MD5);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(TABLE_NAME)
+ .setUpsertConfig(upsertConfig)
+ .setDedupConfig(dedupConfig)
+ .setRoutingConfig(
+ new RoutingConfig(null, null,
RoutingConfig.STRICT_REPLICA_GROUP_INSTANCE_SELECTOR_TYPE, false))
+ .setStreamConfigs(getStreamConfigs())
+ .build();
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ TableConfigUtils.validateUpsertAndDedupConfig(tableConfig, schema);
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
@Test
public void testValidateUpsertConfig() {
Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
index 7340d76b33f..7010bd16032 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
@@ -134,6 +134,7 @@ import org.apache.pinot.spi.utils.CommonConstants.Server;
import
org.apache.pinot.spi.utils.CommonConstants.Server.SegmentCompletionProtocol;
import org.apache.pinot.spi.utils.InstanceTypeUtils;
import org.apache.pinot.spi.utils.NetUtils;
+import org.apache.pinot.spi.utils.PinotMd5Mode;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.sql.parsers.rewriter.QueryRewriterFactory;
import org.slf4j.Logger;
@@ -204,6 +205,8 @@ public abstract class BaseServerStarter implements
ServiceStartable {
PinotInsecureMode.setPinotInInsecureMode(
_serverConf.getProperty(CommonConstants.CONFIG_OF_PINOT_INSECURE_MODE,
false));
+
PinotMd5Mode.setPinotMd5Disabled(_serverConf.getProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
+ PinotMd5Mode.isPinotMd5Disabled()));
String tarCompressionCodecName =
_serverConf.getProperty(CommonConstants.CONFIG_OF_PINOT_TAR_COMPRESSION_CODEC_NAME);
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index f5c9e45b537..88b9bc30520 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -63,6 +63,7 @@ public class CommonConstants {
public static final String DATABASE = "database";
public static final String DEFAULT_DATABASE = "default";
public static final String CONFIG_OF_PINOT_INSECURE_MODE =
"pinot.insecure.mode";
+ public static final String CONFIG_OF_PINOT_MD5_DISABLED =
"pinot.md5.disabled";
@Deprecated
public static final String DEFAULT_PINOT_INSECURE_MODE = "false";
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/PinotMd5Mode.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/PinotMd5Mode.java
new file mode 100644
index 00000000000..eb5bd2d5954
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/PinotMd5Mode.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.utils;
+
+/**
+ * Global runtime switch for disabling MD5-dependent code paths in Pinot.
+ */
+public class PinotMd5Mode {
+ private static volatile boolean _pinotMd5Disabled = readFromSystemProperty();
+
+ private PinotMd5Mode() {
+ }
+
+ public static void setPinotMd5Disabled(boolean pinotMd5Disabled) {
+ _pinotMd5Disabled = pinotMd5Disabled;
+ }
+
+ public static boolean isPinotMd5Disabled() {
+ return _pinotMd5Disabled;
+ }
+
+ // Visible for tests in the same package.
+ static void resetFromSystemProperty() {
+ _pinotMd5Disabled = readFromSystemProperty();
+ }
+
+ private static boolean readFromSystemProperty() {
+ return Boolean.getBoolean(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED);
+ }
+}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/PinotMd5ModeTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/PinotMd5ModeTest.java
new file mode 100644
index 00000000000..f6bcbb894c7
--- /dev/null
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/PinotMd5ModeTest.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.spi.utils;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+
+public class PinotMd5ModeTest {
+ @Test
+ public void testSetterGetter() {
+ try {
+ PinotMd5Mode.setPinotMd5Disabled(true);
+ assertTrue(PinotMd5Mode.isPinotMd5Disabled());
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ assertFalse(PinotMd5Mode.isPinotMd5Disabled());
+ } finally {
+ PinotMd5Mode.setPinotMd5Disabled(false);
+ }
+ }
+
+ @Test
+ public void testResetFromSystemProperty() {
+ String originalValue =
System.getProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED);
+ try {
+ System.setProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED, "true");
+ PinotMd5Mode.resetFromSystemProperty();
+ assertTrue(PinotMd5Mode.isPinotMd5Disabled());
+
+ System.setProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
"false");
+ PinotMd5Mode.resetFromSystemProperty();
+ assertFalse(PinotMd5Mode.isPinotMd5Disabled());
+ } finally {
+ if (originalValue != null) {
+ System.setProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED,
originalValue);
+ } else {
+ System.clearProperty(CommonConstants.CONFIG_OF_PINOT_MD5_DISABLED);
+ }
+ PinotMd5Mode.resetFromSystemProperty();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
