This is an automated email from the ASF dual-hosted git repository.
codope pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 15ca7a3060 [HUDI-4925] Should Force to use ExpressionPayload in
MergeIntoTableCommand (#6355)
add c73b3bdf7c [HUDI-4850] Add incremental source from GCS to Hudi (#6665)
No new revisions were added by this update.
Summary of changes:
hudi-utilities/pom.xml | 17 ++
.../sources/GcsEventsHoodieIncrSource.java | 199 ++++++++++++++
.../hudi/utilities/sources/GcsEventsSource.java | 213 +++++++++++++++
.../hudi/utilities/sources/HoodieIncrSource.java | 8 +-
.../helpers/CloudObjectsSelectorCommon.java | 118 +++++++++
.../sources/helpers/CloudStoreIngestionConfig.java | 96 +++++++
.../helpers/IncrSourceCloudStorageHelper.java | 81 ++++++
.../sources/helpers/IncrSourceHelper.java | 27 ++
.../sources/helpers/gcs/FileDataFetcher.java | 54 ++++
.../sources/helpers/gcs/FilePathsFetcher.java | 114 ++++++++
.../sources/helpers/gcs/GcsIngestionConfig.java | 27 +-
.../sources/helpers/gcs/MessageBatch.java | 22 +-
.../sources/helpers/gcs/MessageValidity.java | 55 ++++
.../sources/helpers/gcs/MetadataMessage.java | 101 +++++++
.../sources/helpers/gcs/PubsubMessagesFetcher.java | 115 ++++++++
.../utilities/sources/helpers/gcs/QueryInfo.java | 94 +++++++
.../sources/TestGcsEventsHoodieIncrSource.java | 289 +++++++++++++++++++++
.../utilities/sources/TestGcsEventsSource.java | 261 +++++++++++++++++++
.../delta-streamer-config/gcs-metadata.avsc | 108 ++++++++
pom.xml | 2 +
20 files changed, 1976 insertions(+), 25 deletions(-)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsHoodieIncrSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/GcsEventsSource.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/CloudObjectsSelectorCommon.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/CloudStoreIngestionConfig.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceCloudStorageHelper.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/FileDataFetcher.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/FilePathsFetcher.java
copy
hudi-common/src/main/java/org/apache/hudi/common/engine/TaskContextSupplier.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/GcsIngestionConfig.java
(56%)
copy
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java
=>
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MessageBatch.java
(65%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MessageValidity.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/MetadataMessage.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/PubsubMessagesFetcher.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/QueryInfo.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsHoodieIncrSource.java
create mode 100644
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGcsEventsSource.java
create mode 100644
hudi-utilities/src/test/resources/delta-streamer-config/gcs-metadata.avsc
