This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1f24630 Cleanup: Make `jobType` case insensitive. (#7244)
1f24630 is described below
commit 1f2463016cd1e38443ec627a9b264de18a41ee0b
Author: Mayank Shrivastava <[email protected]>
AuthorDate: Tue Aug 3 13:35:14 2021 -0700
Cleanup: Make `jobType` case insensitive. (#7244)
The `jobType` field inside of SegmentGenerationSpec is case sensitive.
Converting it into case insensitive for better usability. Also, cleaned
up comments to be in sync with all values of `PinotIngestionJobType`.
---
.../docker/ingestion-job-specs/airlineStats.yaml | 4 ++-
.../docker/ingestion-job-specs/baseballStats.yaml | 4 ++-
.../spi/ingestion/batch/IngestionJobLauncher.java | 36 ++++++++++++++++++++--
.../batch/spec/SegmentGenerationJobSpec.java | 19 +++---------
.../batch/airlineStats/hadoopIngestionJobSpec.yaml | 4 ++-
.../batch/airlineStats/ingestionJobSpec.yaml | 4 ++-
.../batch/airlineStats/sparkIngestionJobSpec.yaml | 4 ++-
.../batch/baseballStats/ingestionJobSpec.yaml | 4 ++-
.../batch/baseballStats/sparkIngestionJobSpec.yaml | 4 ++-
.../batch/dimBaseballTeams/ingestionJobSpec.yaml | 4 ++-
.../ingestionJobComplexTypeHandlingSpec.yaml | 4 ++-
.../batch/githubEvents/ingestionJobSpec.yaml | 4 ++-
.../batch/githubEvents/sparkIngestionJobSpec.yaml | 4 ++-
.../batch/starbucksStores/ingestionJobSpec.yaml | 4 ++-
14 files changed, 73 insertions(+), 30 deletions(-)
diff --git
a/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
b/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
index 9cbb9a8..b213f16 100644
--- a/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
+++ b/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
b/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
index 9d79118..2907596 100644
--- a/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
+++ b/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
index 63ffcd7..511c1ca 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/IngestionJobLauncher.java
@@ -18,12 +18,14 @@
*/
package org.apache.pinot.spi.ingestion.batch;
+import com.fasterxml.jackson.annotation.JsonCreator;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
@@ -95,7 +97,7 @@ public class IngestionJobLauncher {
new Yaml().dump(spec, sw);
LOGGER.info("SegmentGenerationJobSpec: \n{}", sw.toString());
ExecutionFrameworkSpec executionFramework =
spec.getExecutionFrameworkSpec();
- PinotIngestionJobType jobType =
PinotIngestionJobType.valueOf(spec.getJobType());
+ PinotIngestionJobType jobType =
PinotIngestionJobType.fromString(spec.getJobType());
switch (jobType) {
case SegmentCreation:
kickoffIngestionJob(spec,
executionFramework.getSegmentGenerationJobRunnerClassName());
@@ -145,7 +147,35 @@ public class IngestionJobLauncher {
}
}
- enum PinotIngestionJobType {
- SegmentCreation, SegmentTarPush, SegmentUriPush, SegmentMetadataPush,
SegmentCreationAndTarPush, SegmentCreationAndUriPush,
SegmentCreationAndMetadataPush,
+ /**
+ * Ingestion Job type Enum.
+ */
+ public enum PinotIngestionJobType {
+ SegmentCreation,
+ SegmentTarPush,
+ SegmentUriPush,
+ SegmentMetadataPush,
+ SegmentCreationAndTarPush,
+ SegmentCreationAndUriPush,
+ SegmentCreationAndMetadataPush;
+
+ private static final Map<String, PinotIngestionJobType> VALUE_MAP = new
HashMap<>();
+
+ static {
+ for (PinotIngestionJobType jobType : PinotIngestionJobType.values()) {
+ // Use case-insensitive naming.
+ VALUE_MAP.put(jobType.name().toLowerCase(), jobType);
+ }
+ }
+
+ @JsonCreator
+ public static PinotIngestionJobType fromString(String name) {
+ PinotIngestionJobType jobType = VALUE_MAP.get(name.toLowerCase());
+
+ if (jobType == null) {
+ throw new IllegalArgumentException("No enum constant for: " + name);
+ }
+ return jobType;
+ }
}
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
index d90541b..061bd4b 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/spec/SegmentGenerationJobSpec.java
@@ -33,15 +33,9 @@ public class SegmentGenerationJobSpec implements
Serializable {
*/
private ExecutionFrameworkSpec _executionFrameworkSpec;
+
/**
- * Supported job types are:
- * 'SegmentCreation'
- * 'SegmentTarPush'
- * 'SegmentUriPush'
- * 'SegmentMetadataPush'
- * 'SegmentCreationAndTarPush'
- * 'SegmentCreationAndUriPush'
- * 'SegmentCreationAndMetadataPush'
+ * Supported job types are {@link
org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher.PinotIngestionJobType}
*/
private String _jobType;
@@ -144,13 +138,8 @@ public class SegmentGenerationJobSpec implements
Serializable {
}
/**
- * Supported job types are:
- * 'SegmentCreation'
- * 'SegmentTarPush'
- * 'SegmentUriPush'
- * 'SegmentCreationAndTarPush'
- * 'SegmentCreationAndUriPush'
- * @param jobType
+ * Set the job type for the ingestion job.
+ * @param jobType Job type for the ingestion job.
*/
public void setJobType(String jobType) {
_jobType = jobType;
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
index 6679765..474d43e 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
@@ -39,12 +39,14 @@ executionFrameworkSpec:
stagingDir: examples/batch/airlineStats/staging
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
index 13527b9..3b2cfa9 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
index c61d8c1..ebc0280 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
@@ -39,12 +39,14 @@ executionFrameworkSpec:
stagingDir: examples/batch/airlineStats/staging
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
index 2d97af3..530d29d 100644
---
a/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
index b0c79dc..2c9f41f 100644
---
a/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
@@ -35,12 +35,14 @@ executionFrameworkSpec:
extraConfigs:
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
index f456f21..c5af760 100644
---
a/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
index 391b58c..035b2c5 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
index c1af83d..aae8492 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
index 47e1dbe..5b528a6 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
@@ -35,12 +35,14 @@ executionFrameworkSpec:
extraConfigs:
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
diff --git
a/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
index fdd80f2..1906de8 100644
---
a/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
@@ -33,12 +33,14 @@ executionFrameworkSpec:
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
# jobType: Pinot ingestion job type.
-# Supported job types are:
+# Supported job types are defined in PinotIngestionJobType class.
# 'SegmentCreation'
# 'SegmentTarPush'
# 'SegmentUriPush'
+# 'SegmentMetadataPush'
# 'SegmentCreationAndTarPush'
# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush
# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
