This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8a3eb43 Add a new table config field for peer segment download.
(#5478)
8a3eb43 is described below
commit 8a3eb4315df59d6d73a4ec11308d30eab1c51042
Author: Ting Chen <[email protected]>
AuthorDate: Tue Jun 2 14:00:47 2020 -0700
Add a new table config field for peer segment download. (#5478)
* Add a new table config field for peer segment download.
* Address feedbacks.
* Update
pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
Co-authored-by: Subbu Subramaniam <[email protected]>
* Update
pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
Co-authored-by: Subbu Subramaniam <[email protected]>
Co-authored-by: Subbu Subramaniam <[email protected]>
---
.../pinot/common/utils/config/TableConfigUtils.java | 18 ++++++++++++++++++
.../common/utils/config/TableConfigSerDeTest.java | 11 +++++++++++
.../table/SegmentsValidationAndRetentionConfig.java | 11 +++++++++++
.../pinot/spi/utils/builder/TableConfigBuilder.java | 7 +++++++
4 files changed, 47 insertions(+)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
index d93b24b..b7b7d52 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
@@ -181,9 +181,15 @@ public class TableConfigUtils {
* Validates the table config with the following rules:
* <ul>
* <li>Text index column must be raw</li>
+ * <li>peerSegmentDownloadScheme in ValidationConfig must be http or
https</li>
* </ul>
*/
public static void validate(TableConfig tableConfig) {
+ validateFieldConfigList(tableConfig);
+ validateValidationConfig(tableConfig);
+ }
+
+ private static void validateFieldConfigList(TableConfig tableConfig) {
List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList();
if (fieldConfigList != null) {
List<String> noDictionaryColumns =
tableConfig.getIndexingConfig().getNoDictionaryColumns();
@@ -201,4 +207,16 @@ public class TableConfigUtils {
}
}
}
+
+ private static void validateValidationConfig(TableConfig tableConfig) {
+ SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
+ if (validationConfig != null) {
+ String peerSegmentDownloadScheme =
validationConfig.getPeerSegmentDownloadScheme();
+ if (peerSegmentDownloadScheme != null) {
+ if (!"http".equalsIgnoreCase(peerSegmentDownloadScheme) &&
!"https".equalsIgnoreCase(peerSegmentDownloadScheme)) {
+ throw new IllegalStateException("Invalid value '" +
peerSegmentDownloadScheme + "' for peerSegmentDownloadScheme. Must be one of
http nor https" );
+ }
+ }
+ }
+ }
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index 4d34012..d2d2ebf 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -243,6 +243,17 @@ public class TableConfigSerDeTest {
checkTableConfigWithUpsertConfig(JsonUtils.stringToObject(tableConfig.toJsonString(),
TableConfig.class));
checkTableConfigWithUpsertConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig)));
}
+ {
+ // with SegmentsValidationAndRetentionConfig
+ TableConfig tableConfig =
tableConfigBuilder.setPeerSegmentDownloadScheme("http").build();
+
checkSegmentsValidationAndRetentionConfig(JsonUtils.stringToObject(tableConfig.toJsonString(),
TableConfig.class));
+
checkSegmentsValidationAndRetentionConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig)));
+ }
+ }
+
+ private void checkSegmentsValidationAndRetentionConfig(TableConfig
tableConfig) {
+ // TODO validate other fields of SegmentsValidationAndRetentionConfig.
+
assertEquals(tableConfig.getValidationConfig().getPeerSegmentDownloadScheme(),
"http");
}
private void checkDefaultTableConfig(TableConfig tableConfig) {
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
index 4e0f3e2..30451ab 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
@@ -39,6 +39,11 @@ public class SegmentsValidationAndRetentionConfig extends
BaseJsonConfig {
private String _segmentAssignmentStrategy;
private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig;
private CompletionConfig _completionConfig;
+ // Possible values can be http or https. If this field is set, a Pinot
server can download segments from peer servers
+ // using the specified download scheme. Both realtime tables and offline
tables can set this field.
+ // For more usage of this field, please refer to this design doc:
+ //
https://cwiki.apache.org/confluence/display/PINOT/By-passing+deep-store+requirement+for+Realtime+segment+completion#By-passingdeep-storerequirementforRealtimesegmentcompletion-EnablebesteffortsegmentuploadinSplitSegmentCommiteranddownloadsegmentfrompeerservers.
+ private String _peerSegmentDownloadScheme;
// Number of replicas per partition of low-level consumers. This config is
used for realtime tables only.
private String _replicasPerPartition;
@@ -152,4 +157,10 @@ public class SegmentsValidationAndRetentionConfig extends
BaseJsonConfig {
public int getReplicasPerPartitionNumber() {
return Integer.parseInt(_replicasPerPartition);
}
+
+ public String getPeerSegmentDownloadScheme() { return
_peerSegmentDownloadScheme; }
+
+ public void setPeerSegmentDownloadScheme(String peerSegmentDownloadScheme) {
+ _peerSegmentDownloadScheme = peerSegmentDownloadScheme;
+ }
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
index 63cc43c..dbd0c03 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
@@ -64,6 +64,7 @@ public class TableConfigBuilder {
private String _segmentPushFrequency;
private String _segmentPushType = DEFAULT_SEGMENT_PUSH_TYPE;
private String _segmentAssignmentStrategy =
DEFAULT_SEGMENT_ASSIGNMENT_STRATEGY;
+ private String _peerSegmentDownloadScheme;
private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig;
private CompletionConfig _completionConfig;
@@ -287,6 +288,11 @@ public class TableConfigBuilder {
return this;
}
+ public TableConfigBuilder setPeerSegmentDownloadScheme(String
peerSegmentDownloadScheme) {
+ _peerSegmentDownloadScheme = peerSegmentDownloadScheme;
+ return this;
+ }
+
public TableConfig build() {
// Validation config
SegmentsValidationAndRetentionConfig validationConfig = new
SegmentsValidationAndRetentionConfig();
@@ -301,6 +307,7 @@ public class TableConfigBuilder {
validationConfig.setCompletionConfig(_completionConfig);
validationConfig.setSchemaName(_schemaName);
validationConfig.setReplication(_numReplicas);
+ validationConfig.setPeerSegmentDownloadScheme(_peerSegmentDownloadScheme);
if (_isLLC) {
validationConfig.setReplicasPerPartition(_numReplicas);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
