This is an automated email from the ASF dual-hosted git repository.
pwason pushed a change to branch release-0.14.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 9bc6a28010c [MINOR] Fix build on master (#9452)
new be3a7004cf8 [HUDI-6587] Check incomplete commit for time travel query
(#9280)
new d600e98de63 [HUDI-6476][FOLLOW-UP] Path filter by FileStatus to avoid
additional fs request (#9366)
new 544e999c005 [MINOR] StreamerUtil#getTableConfig should check whether
hoodie.properties exists (#9464)
new 6a6bfd7c1e0 [MINOR] Close record readers after use during tests (#9457)
new 0ea1f1b68cb [HUDI-6156] Prevent leaving tmp file in timeline, delete
tmp file when rename throw exception (#9483)
new 2127d3d2c4a [HUDI-6683][FOLLOW-UP] Json & Avro Kafka Source Minor
Refactor & Added null Kafka Key test cases (#9459)
new 18f04344441 [HUDI-6733] Add flink-metrics-dropwizard to flink bundle
(#9499)
new 1ff0a7f2eb1 [HUDI-6731] BigQuerySyncTool: add flag to allow for read
optimized sync for MoR tables (#9488)
new ff6b70f5458 [HUDI-6729] Fix get partition values from path for
non-string type partition column (#9484)
new 5f4bcc8f434 [HUDI-6692] Don't default to bulk insert on nonpkless
table if recordkey is omitted (#9444)
new 55855cd6888 [HUDI-6549] Add support for comma separated path format
for spark.read.load (#9503)
new df90640116c [HUDI-4115] Adding support for schema while loading spark
dataset in S3/GCS source (#9502)
new 0b4c95cdad0 [HUDI-6621] Fix downgrade handler for 0.14.0 (#9467)
new 802d75b285b [HUDI-6718] Check Timeline Before Transitioning Inflight
Clean in Multiwriter Scenario (#9468)
new 8d0e813967a [HUDI-6741] Timeline server bug when multiple tables
registered with metadata table enabled (#9511)
new 1c16d60fef9 [HUDI-6735] Adding support for snapshotLoadQuerySplitter
for incremental sources. (#9501)
new a7690eca670 [HUDI-6445] Triage ci flakiness and some test fies (#9534)
new 0d8c34f24da [HUDI-6754] Fix record reader tests in hudi-hadoop-mr
(#9535)
new 256957a689e [HUDI-6681] Ensure MOR Column Stats Index skips reading
filegroups correctly (#9422)
new f4b139a0556 [MINOR] Add write operation in alter schema commit
metadata (#9509)
new 5e3bf05b282 [MINOR] Add detail exception when instant transition state
(#9476)
new 3eb6de6d00b [HUDI-4631] Adding retries to spark datasource writes on
conflict failures (#6854)
new a4f542931c1 [MINOR] Modify return type description (#9479)
new 2009b0f4466 [HUDI-6726] Fix connection leaks related to file reader
and iterator close (#9539)
new 89a3443173d [MINOR] Fix AWS refactor bug by adding skipTableArchive
arg (#9563)
new eed034b5c82 [HUDI-6758] Detecting and skipping Spurious log blocks
with MOR reads (#9545)
new 2aaf4027110 [MINOR] Fixing warn log with auto key gen (#9547)
new db2129ebb62 [HUDI-3727] Add metrics for async indexer (#9559)
new 9be80c7bc03 [HUDI-6445] Fixing metrics to use IN-MEMORY type in tests
(#9543)
new d995bb8262c [HUDI-6763] Optimize collect calls (#9561)
The 30 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 1 +
.../client/embedded/EmbeddedTimelineService.java | 2 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 16 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 6 +
.../org/apache/hudi/io/HoodieAppendHandle.java | 14 +-
.../hudi/metadata/HoodieMetadataWriteUtils.java | 1 -
.../java/org/apache/hudi/table/HoodieTable.java | 4 +-
.../table/action/clean/CleanActionExecutor.java | 14 +-
.../table/action/commit/HoodieMergeHelper.java | 5 +-
.../table/action/index/RunIndexActionExecutor.java | 16 +-
.../table/upgrade/SixToFiveDowngradeHandler.java | 53 ++-
.../table/upgrade/SupportsUpgradeDowngrade.java | 3 +
.../io/storage/TestHoodieHFileReaderWriter.java | 10 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 1 +
.../table/upgrade/FlinkUpgradeDowngradeHelper.java | 7 +
.../table/upgrade/JavaUpgradeDowngradeHelper.java | 7 +
.../hudi/client/TestJavaHoodieBackedMetadata.java | 16 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 185 ++++----
.../testutils/HoodieJavaClientTestHarness.java | 140 +++---
.../hudi/testutils/TestHoodieMetadataBase.java | 6 +-
.../SparkHoodieBackedTableMetadataWriter.java | 3 +-
.../commit/BaseSparkCommitActionExecutor.java | 14 +-
.../table/upgrade/SparkUpgradeDowngradeHelper.java | 7 +
.../functional/TestHoodieBackedMetadata.java | 18 +-
.../client/functional/TestHoodieMetadataBase.java | 6 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 63 ++-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 10 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 5 +
.../bootstrap/index/HFileBootstrapIndex.java | 8 +-
.../hudi/common/fs/HoodieWrapperFileSystem.java | 14 +-
.../hudi/common/table/TableSchemaResolver.java | 5 +-
.../table/log/AbstractHoodieLogRecordReader.java | 169 ++++++-
.../table/log/block/HoodieHFileDataBlock.java | 23 +-
.../common/table/log/block/HoodieLogBlock.java | 2 +-
.../table/timeline/HoodieActiveTimeline.java | 2 +-
.../hudi/common/table/timeline/TimelineUtils.java | 30 +-
.../common/table/view/FileSystemViewManager.java | 27 +-
.../common/table/view/TableFileSystemView.java | 4 +-
.../hudi/common/util/queue/SimpleExecutor.java | 6 +-
.../hudi/exception/HoodieTimeTravelException.java | 11 +-
.../hudi/io/storage/HoodieAvroHFileReader.java | 173 ++++++--
.../apache/hudi/io/storage/HoodieHFileUtils.java | 24 +-
.../metadata/FileSystemBackedTableMetadata.java | 95 ++--
.../hudi/metadata/HoodieBackedTableMetadata.java | 4 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 3 +-
.../common/functional/TestHoodieLogFormat.java | 143 +++++-
.../java/org/apache/hudi/util/StreamerUtil.java | 2 +-
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 6 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 8 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 14 +-
.../realtime/AbstractRealtimeRecordReader.java | 1 -
.../hive/TestHoodieCombineHiveInputFormat.java | 23 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 6 +
.../realtime/TestHoodieRealtimeRecordReader.java | 47 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 63 ++-
.../apache/hudi/AutoRecordKeyGenerationUtils.scala | 7 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 50 +--
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 42 +-
.../apache/hudi/functional/TestBootstrapRead.java | 2 +-
.../TestDataSkippingWithMORColstats.java | 483 +++++++++++++++++++++
.../functional/TestNewHoodieParquetFileFormat.java | 4 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 84 +++-
.../TestGetPartitionValuesFromPath.scala | 53 +++
.../functional/TestSixToFiveDowngradeHandler.scala | 142 ++++++
.../hudi/functional/TestTimeTravelQuery.scala | 182 ++++----
.../hudi/command/Spark30AlterTableCommand.scala | 1 +
.../hudi/command/Spark31AlterTableCommand.scala | 1 +
.../spark/sql/hudi/command/AlterTableCommand.scala | 1 +
.../sources/GcsEventsHoodieIncrSource.java | 5 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 17 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 2 +-
.../sources/S3EventsHoodieIncrSource.java | 5 +-
.../sources/SnapshotLoadQuerySplitter.java | 78 ++++
.../utilities/sources/helpers/AvroConvertor.java | 11 +-
.../sources/helpers/CloudDataFetcher.java | 6 +-
.../helpers/CloudObjectsSelectorCommon.java | 28 +-
.../sources/helpers/CloudStoreIngestionConfig.java | 12 +
.../hudi/utilities/sources/helpers/QueryInfo.java | 12 +
.../utilities/sources/TestAvroKafkaSource.java | 30 ++
.../sources/TestGcsEventsHoodieIncrSource.java | 34 +-
.../utilities/sources/TestHoodieIncrSource.java | 22 +-
.../utilities/sources/TestJsonKafkaSource.java | 14 +
.../sources/TestS3EventsHoodieIncrSource.java | 28 +-
.../helpers/TestCloudObjectsSelectorCommon.java | 18 +
.../helpers/TestSnapshotQuerySplitterImpl.java} | 42 +-
.../utilities/testutils/UtilitiesTestBase.java | 9 +
.../test/resources/schema/sample_data_schema.avsc | 16 +-
.../src/test/resources/schema/sample_gcs_data.avsc | 20 +-
packaging/hudi-flink-bundle/pom.xml | 1 +
89 files changed, 2289 insertions(+), 719 deletions(-)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieSavepointException.java
=>
hudi-common/src/main/java/org/apache/hudi/exception/HoodieTimeTravelException.java
(83%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestDataSkippingWithMORColstats.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestGetPartitionValuesFromPath.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSixToFiveDowngradeHandler.scala
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SnapshotLoadQuerySplitter.java
copy
hudi-utilities/src/{main/java/org/apache/hudi/utilities/sources/helpers/CloudDataFetcher.java
=>
test/java/org/apache/hudi/utilities/sources/helpers/TestSnapshotQuerySplitterImpl.java}
(51%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BucketType.java
=> hudi-utilities/src/test/resources/schema/sample_data_schema.avsc (81%)
copy hudi-common/src/test/resources/simple-test.avsc =>
hudi-utilities/src/test/resources/schema/sample_gcs_data.avsc (78%)
