This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-0.12.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 48dd28fa86 [HUDI-4760] Fixing repeated trigger of data file creations
w/ clustering (#6561)
add cd44adbee3 [HUDI-4584] Cleaning up Spark utilities (#6351)
add a496c8dd15 fix test
new 0538630811 [HUDI-4914] Managed memory weight should be set when sort
clustering is enabled (#6792)
new f2360894b5 [HUDI-4910] Fix unknown variable or type "Cast" (#6778)
new 6685101262 [HUDI-4918] Fix bugs about when trying to show the non
-existing key from env, NullPointException occurs. (#6794)
new efedf5dd5f [HUDI-4718] Add Kerberos kinit command support. (#6719)
new 070a395c5e [HUDI-4902] Set default partitioner for SIMPLE BUCKET index
(#6759)
new f1661187f5 [MINOR] Update PR template with documentation update (#6748)
new 00d80215df [HUDI-4907] Prevent single commit multi instant issue
(#6766)
new 3e82f13f6e [HUDI-4923] Fix flaky
TestHoodieReadClient.testReadFilterExistAfterBulkInsertPrepped (#6801)
new 1abdb57877 [HUDI-4848] Fixing repair deprecated partition tool (#6731)
new dcec30f14c [HUDI-4913] Fix HoodieSnapshotExporter for writing to a
different S3 bucket or FS (#6785)
new 739da48f1c [HUDI-4453] Fix schema to include partition columns in
bootstrap operation (#6676)
new e2a9d3b790 [HUDI-2780] Fix the issue of Mor log skipping complete
blocks when reading data (#4015)
new b8b0256095 [HUDI-4924] Auto-tune dedup parallelism (#6802)
new c9dd84088e [HUDI-4687] Avoid setAccessible which breaks strong
encapsulation (#6657)
new 1de1dfcd84 [MINOR] fixing validate async operations to poll completed
clean instances (#6814)
new 9a3fcc8456 [HUDI-4734] Deltastreamer table config change validation
(#6753)
new d201c8420a [HUDI-4934] Revert batch clean files (#6813)
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 9 +
.../commands/KerberosAuthenticationCommand.java | 60 ++++
.../apache/hudi/cli/commands/SparkEnvCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 23 +-
.../org/apache/hudi/config/HoodieLayoutConfig.java | 11 +-
.../action/clean/CleanPlanActionExecutor.java | 11 +-
.../hudi/table/action/clean/CleanPlanner.java | 215 +++++++-------
.../table/action/commit/HoodieWriteHelper.java | 5 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 17 ++
.../HoodieSparkBootstrapSchemaProvider.java | 33 +--
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 5 +
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 2 +-
.../table/upgrade/SparkUpgradeDowngradeHelper.java | 4 +-
.../org/apache/hudi/AvroConversionUtils.scala | 9 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 206 +------------
.../scala/org/apache/hudi/util/PathUtils.scala | 84 ++++++
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 2 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 152 +++++++---
.../spark/sql/HoodieCatalystPlansUtils.scala | 5 -
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 2 +-
.../apache/hudi/client/TestHoodieReadClient.java | 15 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 14 +-
.../org/apache/hudi/data/TestHoodieJavaRDD.java} | 24 +-
...dieSparkCopyOnWriteTableArchiveWithReplace.java | 4 +-
hudi-common/pom.xml | 5 +
.../org/apache/hudi/common/data/HoodieData.java | 9 +-
.../apache/hudi/common/data/HoodieListData.java | 5 +
.../hudi/common/table/log/HoodieLogFileReader.java | 15 +-
.../table/view/AbstractTableFileSystemView.java | 16 +-
.../table/view/PriorityBasedFileSystemView.java | 5 -
.../view/RemoteHoodieTableFileSystemView.java | 12 -
.../common/table/view/TableFileSystemView.java | 14 +-
.../hudi/common/util/ObjectSizeCalculator.java | 321 +--------------------
.../hudi/common/data/TestHoodieListData.java | 8 +
.../common/functional/TestHoodieLogFormat.java | 79 +++--
.../hudi/common/util/TestObjectSizeCalculator.java | 102 +++++++
.../hudi/sink/StreamWriteOperatorCoordinator.java | 2 +-
.../hudi/sink/clustering/ClusteringOperator.java | 39 +--
.../org/apache/hudi/sink/meta/CkpMetadata.java | 6 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 10 +-
.../apache/hudi/sink/ITTestDataStreamWrite.java | 12 +
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 3 +-
.../dag/nodes/ValidateAsyncOperations.java | 2 +-
.../org/apache/hudi/integ/ITTestHoodieSanity.java | 4 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 4 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 7 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 2 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 3 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 3 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 55 ----
.../sql/hudi/streaming/HoodieStreamSource.scala | 2 +-
.../hudi/command/payload/ExpressionCodeGen.scala | 5 +-
.../procedures/RunClusteringProcedure.scala | 13 +-
.../TestConvertFilterToCatalystExpression.scala | 15 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 11 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 57 ----
.../scala/org/apache/hudi/util/TestPathUtils.scala | 85 ++++++
.../spark/sql/hudi/TestMergeIntoTable2.scala | 40 +++
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 4 -
.../apache/spark/sql/adapter/Spark2Adapter.scala | 11 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 4 -
.../spark/sql/adapter/BaseSpark3Adapter.scala | 8 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 64 ++--
.../hudi/utilities/deltastreamer/DeltaSync.java | 4 +-
.../deltastreamer/HoodieDeltaStreamer.java | 5 +
.../functional/TestHoodieDeltaStreamer.java | 58 +++-
.../utilities/sources/TestJsonKafkaSource.java | 2 +-
packaging/hudi-flink-bundle/pom.xml | 5 +
packaging/hudi-hadoop-mr-bundle/pom.xml | 5 +
packaging/hudi-hive-sync-bundle/pom.xml | 5 +
packaging/hudi-integ-test-bundle/pom.xml | 5 +
packaging/hudi-kafka-connect-bundle/pom.xml | 5 +
packaging/hudi-presto-bundle/pom.xml | 5 +
packaging/hudi-spark-bundle/pom.xml | 5 +
packaging/hudi-trino-bundle/pom.xml | 5 +
packaging/hudi-utilities-bundle/pom.xml | 5 +
pom.xml | 7 +
78 files changed, 1053 insertions(+), 1046 deletions(-)
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/KerberosAuthenticationCommand.java
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala
copy
hudi-client/{hudi-client-common/src/test/java/org/apache/hudi/optimize/TestHilbertCurveUtils.java
=>
hudi-spark-client/src/test/java/org/apache/hudi/data/TestHoodieJavaRDD.java}
(62%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala
