This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 63ceb0c Adding pinot-spi module and moving record reader interface
(#4869)
add 3e7cbfa Pinot Ingestion Jobs Refactor with Pinot Spark Support (#4800)
No new revisions were added by this update.
Summary of changes:
docs/pinot_hadoop.rst | 104 ++++++++++++
pinot-hadoop/README.md | 78 ---------
.../pinot-hadoop}/README.md | 6 +-
.../pinot-hadoop}/pom.xml | 9 +-
.../pinot/hadoop/PinotHadoopJobLauncher.java | 30 ++--
.../pinot/hadoop/io/CombineAvroKeyInputFormat.java | 0
.../org/apache/pinot/hadoop/io/FileHandler.java | 4 +-
.../pinot/hadoop/io/JsonPinotOutputFormat.java | 0
.../apache/pinot/hadoop/io/PinotOutputFormat.java | 0
.../org/apache/pinot/hadoop/io/PinotRecord.java | 0
.../pinot/hadoop/io/PinotRecordSerialization.java | 0
.../apache/pinot/hadoop/io/PinotRecordWriter.java | 2 +-
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 108 ++-----------
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 129 ++++++---------
.../pinot/hadoop/job/InternalConfigConstants.java | 0
.../hadoop/job/mappers/SegmentCreationMapper.java | 2 +-
.../job/mappers/SegmentPreprocessingMapper.java | 8 +-
.../job/partitioners/GenericPartitioner.java | 8 +-
.../job/partitioners/PartitionFunctionFactory.java | 5 +-
.../job/reducers/SegmentPreprocessingReducer.java | 1 +
.../utils/PinotHadoopJobPreparationHelper.java | 25 +--
.../hadoop/io/DeleteExtraPushedSegmentsTest.java | 4 +-
.../pinot/hadoop/io/PinotOutputFormatTest.java | 0
.../hadoop/job/mappers/SegmentCreationTest.java | 0
.../pinot-ingestion-common}/pom.xml | 78 +++++----
.../pinot/ingestion/common}/ControllerRestApi.java | 2 +-
.../common}/DefaultControllerRestApi.java | 63 +++++---
.../ingestion/common}/JobConfigConstants.java | 16 +-
.../ingestion/common/PinotIngestionJobType.java | 14 +-
.../pinot/ingestion/jobs}/BaseSegmentJob.java | 55 ++++++-
.../pinot/ingestion/jobs/SegmentCreationJob.java | 169 +++++++++++++++++++
.../ingestion/jobs/SegmentPreprocessingJob.java | 115 +++++++++++++
.../pinot/ingestion/jobs}/SegmentTarPushJob.java | 30 ++--
.../pinot/ingestion/jobs}/SegmentUriPushJob.java | 31 ++--
.../ingestion}/utils/JobPreparationHelper.java | 20 +--
.../pinot/ingestion}/utils/PushLocation.java | 5 +-
.../pinot-spark}/README.md | 6 +-
.../pinot-spark}/pom.xml | 71 ++++++--
.../apache/pinot/spark/PinotSparkJobLauncher.java | 44 +++--
.../spark/jobs/SparkSegmentCreationFunction.java | 114 +++----------
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 180 +++++++++++++++++++++
.../pinot/spark/jobs/SparkSegmentTarPushJob.java | 88 ++++++++++
.../pinot/spark/jobs/SparkSegmentUriPushJob.java | 71 ++++++++
.../utils/PinotSparkJobPreparationHelper.java | 36 +----
{pinot-connectors => pinot-ingestion-jobs}/pom.xml | 16 +-
...mentBuildPushOfflineClusterIntegrationTest.java | 16 +-
pom.xml | 14 +-
47 files changed, 1172 insertions(+), 605 deletions(-)
delete mode 100644 pinot-hadoop/README.md
copy {pinot-connectors/pinot-connector-kafka-base =>
pinot-ingestion-jobs/pinot-hadoop}/README.md (77%)
copy {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/pom.xml (96%)
copy {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/PinotHadoopJobLauncher.java
(72%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/CombineAvroKeyInputFormat.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/FileHandler.java
(97%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/JsonPinotOutputFormat.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecord.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecordSerialization.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecordWriter.java
(98%)
rename
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
=>
pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
(57%)
rename
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java
=>
pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
(77%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/InternalConfigConstants.java
(100%)
copy {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
(99%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
(94%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/partitioners/GenericPartitioner.java
(87%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java
(98%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/reducers/SegmentPreprocessingReducer.java
(99%)
copy
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/JobPreparationHelper.java
=>
pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/PinotHadoopJobPreparationHelper.java
(64%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/io/DeleteExtraPushedSegmentsTest.java
(97%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/io/PinotOutputFormatTest.java
(100%)
rename {pinot-hadoop =>
pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationTest.java
(100%)
copy {pinot-hadoop-filesystem =>
pinot-ingestion-jobs/pinot-ingestion-common}/pom.xml (70%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/ControllerRestApi.java
(96%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/DefaultControllerRestApi.java
(78%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/JobConfigConstants.java
(79%)
copy
pinot-common/src/main/java/org/apache/pinot/common/exception/InvalidConfigException.java
=>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/PinotIngestionJobType.java
(79%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/BaseSegmentJob.java
(73%)
create mode 100644
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentCreationJob.java
create mode 100644
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentPreprocessingJob.java
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/SegmentTarPushJob.java
(73%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/SegmentUriPushJob.java
(60%)
copy {pinot-hadoop/src/main/java/org/apache/pinot/hadoop =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion}/utils/JobPreparationHelper.java
(72%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop =>
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion}/utils/PushLocation.java
(92%)
copy {pinot-connectors/pinot-connector-kafka-base =>
pinot-ingestion-jobs/pinot-spark}/README.md (77%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-spark}/pom.xml (76%)
rename
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/PinotHadoopJobLauncher.java
=>
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/PinotSparkJobLauncher.java
(60%)
rename
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
=>
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
(76%)
create mode 100644
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
create mode 100644
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentTarPushJob.java
create mode 100644
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentUriPushJob.java
rename
pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/JobPreparationHelper.java
=>
pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/utils/PinotSparkJobPreparationHelper.java
(55%)
copy {pinot-connectors => pinot-ingestion-jobs}/pom.xml (84%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
