This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 18051eb Clear legacy configs when converting to new TableConfig.
(#7071)
18051eb is described below
commit 18051eb9fd7f28e2a4c84ae689345205023e9cd0
Author: Mayank Shrivastava <[email protected]>
AuthorDate: Thu Jun 17 21:40:53 2021 -0700
Clear legacy configs when converting to new TableConfig. (#7071)
We moved to StreamIngestionConfig and BatchIngestionConfig recently.
In this PR, we clear the legacy configs after converting to the
new TableConfig in the utility `convertFromLegacyTableConfig`.
Added unit test for the same.
---
.../common/utils/config/TableConfigUtils.java | 48 ++++++++++++++++------
.../common/utils/config/TableConfigUtilsTest.java | 8 ++++
2 files changed, 44 insertions(+), 12 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
index 8dc39e4..dac88a9 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
@@ -224,8 +224,6 @@ public class TableConfigUtils {
* <li>The conversion happens in-place, the specified tableConfig is
mutated in-place.</li>
* </ul>
*
- * TODO: We should clear the values from deprecated configs after conversion.
- *
* @param tableConfig Input table config.
*/
public static void convertFromLegacyTableConfig(TableConfig tableConfig) {
@@ -235,34 +233,60 @@ public class TableConfigUtils {
(ingestionConfig != null) ? ingestionConfig.getBatchIngestionConfig()
: null;
SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
+ String segmentPushType = validationConfig.getSegmentPushType();
+ String segmentPushFrequency = validationConfig.getSegmentPushFrequency();
+
if (batchIngestionConfig == null) {
- batchIngestionConfig = new BatchIngestionConfig(null,
validationConfig.getSegmentPushType(),
- validationConfig.getSegmentPushFrequency());
+ // Only create the config if any of the deprecated config is not null.
+ if (segmentPushType != null || segmentPushFrequency != null) {
+ batchIngestionConfig = new BatchIngestionConfig(null, segmentPushType,
segmentPushFrequency);
+ }
} else {
// This should not happen typically, but since we are in repair mode,
might as well cover this corner case.
if (batchIngestionConfig.getSegmentIngestionType() == null) {
-
batchIngestionConfig.setSegmentIngestionType(validationConfig.getSegmentPushType());
+ batchIngestionConfig.setSegmentIngestionType(segmentPushType);
}
if (batchIngestionConfig.getSegmentIngestionFrequency() == null) {
-
batchIngestionConfig.setSegmentIngestionFrequency(validationConfig.getSegmentPushFrequency());
+
batchIngestionConfig.setSegmentIngestionFrequency(segmentPushFrequency);
}
}
StreamIngestionConfig streamIngestionConfig =
(ingestionConfig != null) ? ingestionConfig.getStreamIngestionConfig()
: null;
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+
if (streamIngestionConfig == null) {
- streamIngestionConfig =
- new
StreamIngestionConfig(Collections.singletonList(tableConfig.getIndexingConfig().getStreamConfigs()));
+ Map<String, String> streamConfigs = indexingConfig.getStreamConfigs();
+
+ // Only set the new config if the deprecated one is set.
+ if (streamConfigs != null && !streamConfigs.isEmpty()) {
+ streamIngestionConfig = new
StreamIngestionConfig(Collections.singletonList(streamConfigs));
+ }
}
if (ingestionConfig == null) {
- ingestionConfig = new IngestionConfig(batchIngestionConfig,
streamIngestionConfig, null, null, null);
+ if (batchIngestionConfig != null || streamIngestionConfig != null) {
+ ingestionConfig = new IngestionConfig(batchIngestionConfig,
streamIngestionConfig, null, null, null);
+ }
} else {
- ingestionConfig.setBatchIngestionConfig(batchIngestionConfig);
- ingestionConfig.setStreamIngestionConfig(streamIngestionConfig);
+ if (batchIngestionConfig != null) {
+ ingestionConfig.setBatchIngestionConfig(batchIngestionConfig);
+ }
+
+ if (streamIngestionConfig != null) {
+ ingestionConfig.setStreamIngestionConfig(streamIngestionConfig);
+ }
+ }
+
+ // Set the new config fields.
+ if (ingestionConfig != null) {
+ tableConfig.setIngestionConfig(ingestionConfig);
}
- tableConfig.setIngestionConfig(ingestionConfig);
+ // Clear the deprecated ones.
+ indexingConfig.setStreamConfigs(null);
+ validationConfig.setSegmentPushFrequency(null);
+ validationConfig.setSegmentPushType(null);
}
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigUtilsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigUtilsTest.java
index 3129af7..0a769a5 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigUtilsTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigUtilsTest.java
@@ -21,6 +21,7 @@ package org.apache.pinot.common.utils.config;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.pinot.spi.config.table.SegmentsValidationAndRetentionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.table.ingestion.BatchIngestionConfig;
@@ -75,6 +76,13 @@ public class TableConfigUtilsTest {
Map<String, String> actualStreamConfigsMap =
tableConfig.getIngestionConfig().getStreamIngestionConfig().getStreamConfigMaps().get(0);
Assert.assertEquals(actualStreamConfigsMap, expectedStreamConfigsMap);
+
+ // Assert that the deprecated fields are cleared.
+ Assert.assertNull(tableConfig.getIndexingConfig().getStreamConfigs());
+
+ SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
+ Assert.assertNull(validationConfig.getSegmentPushFrequency());
+ Assert.assertNull(validationConfig.getSegmentPushType());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
