This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 329a419 Add missing `SegmentMetadataPushJobRunner` config in all the
example yaml files. (#7258)
329a419 is described below
commit 329a419abb51bac4b6978cd657c79b56572a3b79
Author: Mayank Shrivastava <[email protected]>
AuthorDate: Thu Aug 5 20:04:20 2021 -0700
Add missing `SegmentMetadataPushJobRunner` config in all the example yaml
files. (#7258)
The example yaml files are missing the `SegmentMetadataPushJobRunner`
config.
- Added them to complete the examples to be used as reference.
- TBD: Also add an example that uses this config, currently not added as it
needs deep-storage.
---
.../pinot/examples/docker/ingestion-job-specs/airlineStats.yaml | 7 ++-----
.../pinot/examples/docker/ingestion-job-specs/baseballStats.yaml | 7 ++-----
.../src/main/resources/segmentCreationAndTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationAndUriPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationJobSpec.yaml | 1 +
.../src/main/resources/segmentTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentUriPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationAndTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationAndUriPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationJobSpec.yaml | 1 +
.../src/main/resources/segmentTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentUriPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationAndTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationAndUriPushJobSpec.yaml | 1 +
.../src/main/resources/segmentCreationJobSpec.yaml | 1 +
.../src/main/resources/segmentTarPushJobSpec.yaml | 1 +
.../src/main/resources/segmentUriPushJobSpec.yaml | 1 +
pinot-spi/src/test/resources/ingestion_job_spec_template.yaml | 1 +
.../examples/batch/airlineStats/hadoopIngestionJobSpec.yaml | 7 ++-----
.../resources/examples/batch/airlineStats/ingestionJobSpec.yaml | 7 ++-----
.../examples/batch/airlineStats/sparkIngestionJobSpec.yaml | 7 ++-----
.../resources/examples/batch/baseballStats/ingestionJobSpec.yaml | 8 +++-----
.../examples/batch/baseballStats/sparkIngestionJobSpec.yaml | 7 ++-----
.../examples/batch/dimBaseballTeams/ingestionJobSpec.yaml | 7 ++-----
.../batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml | 7 ++-----
.../resources/examples/batch/githubEvents/ingestionJobSpec.yaml | 7 ++-----
.../examples/batch/githubEvents/sparkIngestionJobSpec.yaml | 7 ++-----
.../examples/batch/starbucksStores/ingestionJobSpec.yaml | 7 ++-----
28 files changed, 41 insertions(+), 60 deletions(-)
diff --git
a/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
b/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
index b213f16..107f1f4 100644
--- a/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
+++ b/docker/images/pinot/examples/docker/ingestion-job-specs/airlineStats.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
b/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
index 2907596..0faf661 100644
--- a/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
+++ b/docker/images/pinot/examples/docker/ingestion-job-specs/baseballStats.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
index b8fbf1a..2cb019a 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
jobType: SegmentCreationAndTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
index f0e02e1..0ded4c1 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
jobType: SegmentCreationAndUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationJobSpec.yaml
index a8b667b..8c0a489 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentCreationJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
jobType: SegmentCreation
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentTarPushJobSpec.yaml
index 7ce85af..dfbae89 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
jobType: SegmentTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentUriPushJobSpec.yaml
index cf65805..744e7af 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-hadoop/src/main/resources/segmentUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
jobType: SegmentUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
index 97b2722..f220421 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
jobType: SegmentCreationAndTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
index 91e489c..fd7e8fe 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
jobType: SegmentCreationAndUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationJobSpec.yaml
index 3bc4f50..6611ef9 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentCreationJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
jobType: SegmentCreation
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentTarPushJobSpec.yaml
index 8bc4f79..cbf7025 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
jobType: SegmentTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentUriPushJobSpec.yaml
index 656cba7..4da281c 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/resources/segmentUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
jobType: SegmentUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
index f4dd4b2..f7752d3 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentCreationAndTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
index 73495e6..c36274c 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentCreationAndUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationJobSpec.yaml
index 4389462..e2fa3d4 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentCreationJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentCreation
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentTarPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentTarPushJobSpec.yaml
index 3ccd6e1..5aedb12 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentTarPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentTarPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentTarPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentUriPushJobSpec.yaml
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentUriPushJobSpec.yaml
index 5e985ae..2d352ba 100644
---
a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentUriPushJobSpec.yaml
+++
b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/main/resources/segmentUriPushJobSpec.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentUriPush
inputDirURI: 'file:///path/to/input'
includeFileNamePattern: 'glob:**/*.parquet'
diff --git a/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
b/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
index 23a1e53..e68bec2 100644
--- a/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
+++ b/pinot-spi/src/test/resources/ingestion_job_spec_template.yaml
@@ -22,6 +22,7 @@ executionFrameworkSpec:
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
jobType: SegmentCreationAndTarPush
inputDirURI: 'file:///path/to/input/${ year }/${ month }/${ day }'
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
index 474d43e..15bf4e0 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/hadoopIngestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'hadoop'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.hadoop.HadoopSegmentMetadataPushJobRunner'
# extraConfigs: extra configs for execution framework.
extraConfigs:
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
index 3b2cfa9..99eb073 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/ingestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
index ebc0280..89307bc 100644
---
a/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/airlineStats/sparkIngestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'spark'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
# extraConfigs: extra configs for execution framework.
extraConfigs:
diff --git
a/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
index 530d29d..50404f9 100644
---
a/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/baseballStats/ingestionJobSpec.yaml
@@ -23,14 +23,12 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
+
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
index 2c9f41f..c0efab8 100644
---
a/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/baseballStats/sparkIngestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'spark'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
extraConfigs:
diff --git
a/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
index c5af760..899687e 100644
---
a/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/ingestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
index 035b2c5..04b67c4 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobComplexTypeHandlingSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
index aae8492..f8216f1 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
diff --git
a/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
index 5b528a6..05cb2f9 100644
---
a/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'spark'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentMetadataPushJobRunner'
extraConfigs:
diff --git
a/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
index 1906de8..b6a4046 100644
---
a/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
+++
b/pinot-tools/src/main/resources/examples/batch/starbucksStores/ingestionJobSpec.yaml
@@ -23,14 +23,11 @@ executionFrameworkSpec:
# name: execution framework name
name: 'standalone'
- # segmentGenerationJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentGenerationJobRunner
interface.
+ # Class to use for segment generation and different push types.
segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
-
- # segmentTarPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentTarPushJobRunner interface.
segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
-
- # segmentUriPushJobRunnerClassName: class name implements
org.apache.pinot.spi.batch.ingestion.runner.SegmentUriPushJobRunner interface.
segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
# jobType: Pinot ingestion job type.
# Supported job types are defined in PinotIngestionJobType class.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
