This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 79407b5 Failure to recover after segment reload failure does not
release segment lock (#4882)
add 423692b Pinot ingestion job - Standalone (#4874)
No new revisions were added by this update.
Summary of changes:
.../pinot-hadoop/README.md | 0
.../pinot-hadoop/pom.xml | 2 +-
.../pinot/hadoop/PinotHadoopJobLauncher.java | 0
.../pinot/hadoop/io/CombineAvroKeyInputFormat.java | 0
.../org/apache/pinot/hadoop/io/FileHandler.java | 0
.../pinot/hadoop/io/JsonPinotOutputFormat.java | 0
.../apache/pinot/hadoop/io/PinotOutputFormat.java | 0
.../org/apache/pinot/hadoop/io/PinotRecord.java | 0
.../pinot/hadoop/io/PinotRecordSerialization.java | 0
.../apache/pinot/hadoop/io/PinotRecordWriter.java | 0
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 0
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 0
.../pinot/hadoop/job/InternalConfigConstants.java | 0
.../hadoop/job/mappers/SegmentCreationMapper.java | 0
.../job/mappers/SegmentPreprocessingMapper.java | 0
.../job/partitioners/GenericPartitioner.java | 0
.../job/partitioners/PartitionFunctionFactory.java | 0
.../job/reducers/SegmentPreprocessingReducer.java | 0
.../utils/PinotHadoopJobPreparationHelper.java | 0
.../hadoop/io/DeleteExtraPushedSegmentsTest.java | 0
.../pinot/hadoop/io/PinotOutputFormatTest.java | 0
.../hadoop/job/mappers/SegmentCreationTest.java | 0
.../pinot-ingestion-common/pom.xml | 2 +-
.../pinot/ingestion/common/ControllerRestApi.java | 0
.../ingestion/common/DefaultControllerRestApi.java | 2 +-
.../pinot/ingestion/common/JobConfigConstants.java | 0
.../ingestion/common/PinotIngestionJobType.java | 0
.../pinot/ingestion/jobs/BaseSegmentJob.java | 0
.../pinot/ingestion/jobs/SegmentCreationJob.java | 0
.../ingestion/jobs/SegmentPreprocessingJob.java | 0
.../pinot/ingestion/jobs/SegmentTarPushJob.java | 0
.../pinot/ingestion/jobs/SegmentUriPushJob.java | 0
.../ingestion/utils/JobPreparationHelper.java | 0
.../apache/pinot/ingestion/utils/PushLocation.java | 0
.../pinot-spark/README.md | 0
.../pinot-spark/pom.xml | 2 +-
.../apache/pinot/spark/PinotSparkJobLauncher.java | 0
.../spark/jobs/SparkSegmentCreationFunction.java | 0
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 0
.../pinot/spark/jobs/SparkSegmentTarPushJob.java | 0
.../pinot/spark/jobs/SparkSegmentUriPushJob.java | 0
.../utils/PinotSparkJobPreparationHelper.java | 0
.../pinot-standalone}/pom.xml | 87 ++----
.../apache/pinot/ingestion/common/Constants.java | 12 +-
.../pinot/ingestion/common/PinotClusterSpec.java | 45 +--
.../apache/pinot/ingestion/common/PinotFSSpec.java | 88 ++++++
.../apache/pinot/ingestion/common/PushJobSpec.java | 92 ++++++
.../pinot/ingestion/common/RecordReaderSpec.java | 119 ++++++++
.../ingestion/common/SegmentGenerationJobSpec.java | 223 ++++++++++++++
.../common/SegmentGenerationTaskRunner.java | 134 +++++++++
.../common/SegmentGenerationTaskSpec.java | 121 ++++++++
.../ingestion/common/SegmentNameGeneratorSpec.java | 35 ++-
.../apache/pinot/ingestion/common/TableSpec.java | 72 +++++
.../standalone/SegmentGenerationJobRunner.java | 319 +++++++++++++++++++++
.../standalone/SegmentTarPushJobRunner.java | 151 ++++++++++
.../standalone/SegmentUriPushJobRunner.java | 146 ++++++++++
.../standalone/StandaloneIngestionJobLauncher.java | 86 ++++++
.../segmentCreationAndTarPushJobSpec.yaml | 29 +-
.../segmentCreationAndUriPushJobSpec.yaml | 29 +-
.../src/main/resources/segmentCreationJobSpec.yaml | 26 +-
.../src/main/resources/segmentTarPushJobSpec.yaml | 29 +-
.../src/main/resources/segmentUriPushJobSpec.yaml | 29 +-
.../pom.xml | 5 +-
.../common/utils/FileUploadDownloadClient.java | 10 +
.../apache/pinot/filesystem/PinotFSFactory.java | 28 +-
pom.xml | 2 +-
66 files changed, 1771 insertions(+), 154 deletions(-)
rename {pinot-ingestion-jobs => pinot-batch-ingestion}/pinot-hadoop/README.md
(100%)
copy {pinot-ingestion-jobs => pinot-batch-ingestion}/pinot-hadoop/pom.xml (99%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/PinotHadoopJobLauncher.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/CombineAvroKeyInputFormat.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/FileHandler.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/JsonPinotOutputFormat.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotRecord.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotRecordSerialization.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/io/PinotRecordWriter.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/InternalConfigConstants.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/GenericPartitioner.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/reducers/SegmentPreprocessingReducer.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/PinotHadoopJobPreparationHelper.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/test/java/org/apache/pinot/hadoop/io/DeleteExtraPushedSegmentsTest.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/test/java/org/apache/pinot/hadoop/io/PinotOutputFormatTest.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-hadoop/src/test/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationTest.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/pom.xml (98%)
copy {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/ControllerRestApi.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/DefaultControllerRestApi.java
(99%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/JobConfigConstants.java
(100%)
copy {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/PinotIngestionJobType.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/BaseSegmentJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentCreationJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentPreprocessingJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentTarPushJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentUriPushJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/utils/JobPreparationHelper.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/utils/PushLocation.java
(100%)
rename {pinot-ingestion-jobs => pinot-batch-ingestion}/pinot-spark/README.md
(100%)
rename {pinot-ingestion-jobs => pinot-batch-ingestion}/pinot-spark/pom.xml
(99%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/PinotSparkJobLauncher.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentTarPushJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentUriPushJob.java
(100%)
rename {pinot-ingestion-jobs =>
pinot-batch-ingestion}/pinot-spark/src/main/java/org/apache/pinot/spark/utils/PinotSparkJobPreparationHelper.java
(100%)
rename {pinot-ingestion-jobs/pinot-hadoop =>
pinot-batch-ingestion/pinot-standalone}/pom.xml (73%)
rename
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/PinotIngestionJobType.java
=>
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/Constants.java
(82%)
rename
pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/ControllerRestApi.java
=>
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/PinotClusterSpec.java
(60%)
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/PinotFSSpec.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/PushJobSpec.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/RecordReaderSpec.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/SegmentGenerationJobSpec.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/SegmentGenerationTaskRunner.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/SegmentGenerationTaskSpec.java
copy
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/entity/AnomalyFeedbackIndex.java
=>
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/SegmentNameGeneratorSpec.java
(58%)
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/common/TableSpec.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/standalone/SegmentGenerationJobRunner.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/standalone/SegmentTarPushJobRunner.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/standalone/SegmentUriPushJobRunner.java
create mode 100644
pinot-batch-ingestion/pinot-standalone/src/main/java/org/apache/pinot/ingestion/standalone/StandaloneIngestionJobLauncher.java
copy docker/images/pinot-presto/etc/config.properties =>
pinot-batch-ingestion/pinot-standalone/src/main/resources/segmentCreationAndTarPushJobSpec.yaml
(53%)
copy docker/images/pinot-presto/etc/config.properties =>
pinot-batch-ingestion/pinot-standalone/src/main/resources/segmentCreationAndUriPushJobSpec.yaml
(53%)
copy docker/images/pinot-presto/etc/config.properties =>
pinot-batch-ingestion/pinot-standalone/src/main/resources/segmentCreationJobSpec.yaml
(56%)
copy docker/images/pinot-presto/etc/config.properties =>
pinot-batch-ingestion/pinot-standalone/src/main/resources/segmentTarPushJobSpec.yaml
(53%)
copy docker/images/pinot-presto/etc/config.properties =>
pinot-batch-ingestion/pinot-standalone/src/main/resources/segmentUriPushJobSpec.yaml
(53%)
rename {pinot-ingestion-jobs => pinot-batch-ingestion}/pom.xml (93%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
