This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-0.12.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit 2c1560c4e3 Create release branch for version 0.12.1.
omit 7bc70aa62a Fix errors from cherry-picking
add f7586a9480 Fix errors from cherry-picking
add bd3a74cd80 Create release branch for version 0.12.1.
add 726f0b6b53 [HUDI-4870] Improve compaction config description (#6706)
add 252e96713b [HUDI-4808] Fix HoodieSimpleBucketIndex not consider bucket
num in lo… (#6630)
add 29b214f5e5 [HUDI-4485] Bump spring shell to 2.1.1 in CLI (#6489)
add 7258140200 [minor] following 3304, some code refactoring (#6713)
add 885615af21 [HUDI-4832] Fix drop partition meta sync (#6662)
add 663c0d516a [HUDI-4810] Fix log4j imports to use bridge API (#6710)
add 85dfbc9fa1 [HUDI-4877] Fix
org.apache.hudi.index.bucket.TestHoodieSimpleBucketIndex#testTagLocation not
work correct issue (#6717)
add 56537c80c7 [HUDI-4326] add updateTableSerDeInfo for HiveSyncTool
(#5920)
add be4e003847 [MINOR] fix indent to make build pass (#6721)
add fe72a3ac1b [HUDI-4326] Fix hive sync serde properties (#6722)
add bc0a1a7fef [HUDI-4875] Fix NoSuchTableException when dropping
temporary view after applied HoodieSparkSessionExtension in Spark 3.2 (#6709)
add 9d870fc3b1 [DOCS] Improve the quick start guide for Kafka Connect Sink
(#6708)
add baf2c9a1be [HUDI-4729] Fix file group pending compaction cannot be
queried when query _ro table (#6516)
add 0d55afceb4 [HUDI-3983] Fix ClassNotFoundException when using
hudi-spark-bundle to write table with hbase index (#6715)
add 38ce5eb0a7 [HUDI-4758] Add validations to java spark examples (#6615)
add 76c062fc25 [HUDI-4792] Batch clean files to delete (#6580)
add bf2d74735e [HUDI-4363] Support Clustering row writer to improve
performance (#6046)
add 2fa516b718 [HUDI-4851] Fixing handling of `UTF8String` w/in `InSet`
operator (#6739)
add 723ffc15b5 [HUDI-3901] Correct the description of hoodie.index.type
(#6749)
add 58c8c4f09c [MINOR] Add .mvn directory to gitignore (#6746)
add 932f7ff92d [HUDI-4901] Add avro.version to Flink profiles (#6757)
add 7d4eb9ec33 [HUDI-4559] Support hiveSync command based on Call Produce
Command (#6322)
add b6c021f04c [HUDI-4883] Supporting delete savepoint for MOR (#6744)
add e8ce8faa87 [HUDI-3523] Introduce AddColumnSchemaPostProcessor to
support add columns to the end of a schema (#5031)
add 91e51b8999 Revert "[HUDI-3523] Introduce AddColumnSchemaPostProcessor
to support add columns to the end of a schema (#5031)" (#6768)
add baeec4ad26 [HUDI-3523] Introduce AddPrimitiveColumnSchemaPostProcessor
to support add new primitive column to the end of a schema (#6769)
add 9adc6cd8a1 [HUDI-4903] Fix TestHoodieLogFormat`s minor typo (#6762)
add 75c764ab9e [MINOR] Drastically reducing concurrency level (to avoid CI
flakiness) (#6754)
add 84d2606a49 Update HoodieIndex.java
add 8a969cb0dc [HUDI-4906] Fix the local tests for hudi-flink (#6763)
add 73628d4de7 [HUDI-4899] Fixing compatibility w/ Spark 3.2.2 (#6755)
add f77f7d2b5c [HUDI-4892] Fix hudi-spark3-bundle (#6735)
add 17b357a603 [MINOR] Fix a few typos in HoodieIndex (#6784)
add a9eb86827f [HUDI-4412] Fix multi writer INSERT_OVERWRITE NPE bug
(#6130)
add 8dc75d8514 [MINOR] retain avro's namespace (#6783)
add e33478910b [MINOR] Simple logging fix in LockManager (#6765)
add 7df0bfde92 [HUDI-4433] hudi-cli repair deduplicate not working with
non-partitioned dataset (#6349)
add e34931ac7a [RFC-51][HUDI-3478] Update RFC: CDC support (#6256)
add abc509e12d [HUDI-4915] improve avro serializer/deserializer (#6788)
add d229bfb8bd [HUDI-4830] Fix testNoGlobalConfFileConfigured when add
hudi-defaults.conf in default dir (#6652)
add 48dd28fa86 [HUDI-4760] Fixing repeated trigger of data file creations
w/ clustering (#6561)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2c1560c4e3)
\
N -- N -- N refs/heads/release-0.12.1-rc1 (48dd28fa86)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.gitignore | 1 +
docker/demo/compaction-bootstrap.commands | 30 +--
docker/demo/compaction.commands | 30 +--
docker/demo/sync-validate.commands | 30 +--
hudi-cli/hudi-cli.sh | 4 +-
hudi-cli/pom.xml | 44 +++-
.../hudi/cli/HoodieHistoryFileNameProvider.java | 43 ----
.../java/org/apache/hudi/cli/HoodiePrompt.java | 24 +-
.../org/apache/hudi/cli/HoodieSplashScreen.java | 72 ------
.../src/main/java/org/apache/hudi/cli/Main.java | 7 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 52 +++--
.../apache/hudi/cli/commands/BootstrapCommand.java | 74 +++----
.../apache/hudi/cli/commands/CleansCommand.java | 55 +++--
.../hudi/cli/commands/ClusteringCommand.java | 70 +++---
.../apache/hudi/cli/commands/CommitsCommand.java | 145 ++++++------
.../hudi/cli/commands/CompactionCommand.java | 244 ++++++++++-----------
.../org/apache/hudi/cli/commands/DiffCommand.java | 55 ++---
.../apache/hudi/cli/commands/ExportCommand.java | 36 ++-
.../hudi/cli/commands/FileSystemViewCommand.java | 96 ++++----
.../cli/commands/HDFSParquetImportCommand.java | 49 ++---
.../hudi/cli/commands/HoodieLogFileCommand.java | 55 +++--
.../cli/commands/HoodieSyncValidateCommand.java | 28 ++-
.../apache/hudi/cli/commands/MarkersCommand.java | 23 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 52 +++--
.../apache/hudi/cli/commands/RepairsCommand.java | 102 ++++-----
.../apache/hudi/cli/commands/RollbacksCommand.java | 51 +++--
.../hudi/cli/commands/SavepointsCommand.java | 55 ++---
.../apache/hudi/cli/commands/SparkEnvCommand.java | 22 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 5 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 52 +++--
.../org/apache/hudi/cli/commands/TableCommand.java | 73 +++---
.../apache/hudi/cli/commands/TempViewCommand.java | 22 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 102 +++++----
.../cli/commands/UpgradeOrDowngradeCommand.java | 34 +--
.../org/apache/hudi/cli/commands/UtilsCommand.java | 16 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 8 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 25 +--
.../META-INF/spring/spring-shell-plugin.xml | 28 ---
.../src/main/resources/application.yml | 7 +-
hudi-cli/src/main/resources/banner.txt | 14 ++
.../src/main/resources/log4j2.properties | 15 +-
.../scala/org/apache/hudi/cli/DedupeSparkJob.scala | 6 +-
.../cli/commands/TestArchivedCommitsCommand.java | 29 ++-
.../hudi/cli/commands/TestCleansCommand.java | 21 +-
.../hudi/cli/commands/TestCommitsCommand.java | 70 +++---
.../hudi/cli/commands/TestCompactionCommand.java | 26 ++-
.../apache/hudi/cli/commands/TestDiffCommand.java | 14 +-
.../cli/commands/TestFileSystemViewCommand.java | 33 +--
.../cli/commands/TestHoodieLogFileCommand.java | 29 ++-
.../hudi/cli/commands/TestRepairsCommand.java | 35 +--
.../hudi/cli/commands/TestRollbacksCommand.java | 21 +-
.../hudi/cli/commands/TestSavepointsCommand.java | 19 +-
.../hudi/cli/commands/TestSparkEnvCommand.java | 21 +-
.../apache/hudi/cli/commands/TestStatsCommand.java | 21 +-
.../apache/hudi/cli/commands/TestTableCommand.java | 50 +++--
.../hudi/cli/commands/TestTempViewCommand.java | 26 ++-
.../apache/hudi/cli/commands/TestUtilsCommand.java | 33 +--
.../cli/functional/CLIFunctionalTestHarness.java | 15 +-
.../hudi/cli/integ/ITTestBootstrapCommand.java | 31 +--
.../hudi/cli/integ/ITTestClusteringCommand.java | 41 ++--
.../hudi/cli/integ/ITTestCommitsCommand.java | 38 ++--
.../hudi/cli/integ/ITTestCompactionCommand.java | 76 ++++---
.../cli/integ/ITTestHDFSParquetImportCommand.java | 28 ++-
.../hudi/cli/integ/ITTestMarkersCommand.java | 15 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 92 ++++++--
.../hudi/cli/integ/ITTestSavepointsCommand.java | 40 ++--
.../testutils/HoodieCLIIntegrationTestHarness.java | 22 --
.../hudi/cli/testutils/MockCommandLineInput.java | 57 +++++
...estBase.java => ShellEvaluationResultUtil.java} | 33 +--
.../http/HoodieWriteCommitHttpCallbackClient.java | 4 +-
.../hudi/client/HoodieInternalWriteStatus.java | 17 ++
.../client/transaction/ConcurrentOperation.java | 5 +-
.../hudi/client/transaction/lock/LockManager.java | 2 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 8 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 3 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 6 +-
.../hudi/execution/HoodieLazyInsertIterable.java | 2 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 8 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 21 ++
.../hudi/index/bucket/HoodieSimpleBucketIndex.java | 10 +-
.../hudi/metrics/datadog/DatadogHttpClient.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 4 +-
.../action/clean/CleanPlanActionExecutor.java | 11 +-
.../hudi/table/action/clean/CleanPlanner.java | 214 +++++++++---------
.../action/commit/BaseCommitActionExecutor.java | 3 +-
.../table/action/commit/HoodieWriteHelper.java | 3 +-
.../table/action/savepoint/SavepointHelpers.java | 4 -
...urrentFileWritesConflictResolutionStrategy.java | 49 +++++
.../hudi/testutils/HoodieWriteableTestTable.java | 25 +--
.../hudi/table/action/commit/FlinkWriteHelper.java | 3 +-
.../hudi/table/action/commit/JavaWriteHelper.java | 3 +-
hudi-client/hudi-spark-client/pom.xml | 1 +
.../apache/hudi/client/SparkRDDWriteClient.java | 19 ++
.../MultipleSparkJobExecutionStrategy.java | 188 +++++++++++++---
.../SparkSingleFileSortExecutionStrategy.java | 43 +++-
.../SparkSortAndSizeExecutionStrategy.java | 37 +++-
.../bulkinsert/RDDSpatialCurveSortPartitioner.java | 37 +---
...ws.java => RowSpatialCurveSortPartitioner.java} | 25 ++-
.../SpatialCurveSortPartitionerBase.java | 83 +++++++
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 28 ++-
.../BulkInsertDataInternalWriterHelper.java | 17 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 61 +++++-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 1 +
.../scala/org/apache/hudi/HoodieUnsafeRDD.scala | 0
.../org/apache/hudi/SparkAdapterSupport.scala | 8 +-
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 1 -
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 14 +-
.../functional/TestHoodieBackedTableMetadata.java | 2 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 49 ++++-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 25 ++-
...dieSparkCopyOnWriteTableArchiveWithReplace.java | 4 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 3 +
.../testutils/HoodieSparkWriteableTestTable.java | 11 +
.../src/main/avro/HoodieClusteringGroup.avsc | 10 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 1 -
.../apache/hudi/common/config/HoodieConfig.java | 4 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 4 +
.../apache/hudi/common/model/HoodieFileGroup.java | 4 +
.../org/apache/hudi/common/model/HoodieRecord.java | 2 +
.../hudi/common/table/timeline/TimelineUtils.java | 26 ++-
.../table/view/AbstractTableFileSystemView.java | 76 ++++---
.../table/view/PriorityBasedFileSystemView.java | 5 +
.../view/RemoteHoodieTableFileSystemView.java | 12 +
.../common/table/view/TableFileSystemView.java | 14 +-
.../apache/hudi/common/util/CollectionUtils.java | 10 +-
.../convert/AvroInternalSchemaConverter.java | 31 ++-
.../schema/utils/AvroSchemaEvolutionUtils.java | 4 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 18 --
hudi-common/src/main/resources/hbase-site.xml | 7 -
.../common/functional/TestHoodieLogFormat.java | 6 +-
.../hudi/common/table/TestTimelineUtils.java | 4 +-
.../common/testutils/HoodieTestDataGenerator.java | 1 +
.../hudi/common/testutils/HoodieTestTable.java | 6 +
.../util/TestDFSPropertiesConfiguration.java | 12 +-
.../common/HoodieExampleDataGenerator.java | 59 ++++-
hudi-examples/hudi-examples-flink/pom.xml | 6 +
hudi-examples/hudi-examples-spark/pom.xml | 7 +
.../examples/quickstart/HoodieSparkQuickstart.java | 67 ++++--
.../hudi/hadoop/HoodieROTablePathFilter.java | 13 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 4 +-
.../integ/command/ITTestHoodieSyncCommand.java | 4 +-
hudi-kafka-connect/README.md | 23 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 1 +
.../org/apache/hudi/BaseFileOnlyRelation.scala | 1 -
.../main/scala/org/apache/hudi/DefaultSource.scala | 95 ++++----
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 12 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 33 +--
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 4 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 6 +-
.../hudi/command/procedures/BaseProcedure.scala | 34 +--
.../command/procedures/HiveSyncProcedure.scala | 107 +++++++++
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../TestHoodieSparkMergeOnReadTableClustering.java | 55 +++--
.../org/apache/hudi/TestDataSkippingUtils.scala | 29 ++-
.../apache/hudi/functional/TestCOWDataSource.scala | 12 +
.../hudi/functional/TestLayoutOptimization.scala | 37 +++-
.../org/apache/spark/sql/hudi/TestDropTable.scala | 23 +-
.../hudi/TestQueryMergeOnReadOptimizedTable.scala | 82 +++++++
.../HoodieBulkInsertDataInternalWriter.java | 1 +
.../apache/spark/sql/adapter/Spark2Adapter.scala | 15 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 20 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 17 +-
.../catalyst/encoders/DummyExpressionHolder.scala | 16 +-
.../HoodieBulkInsertDataInternalWriter.java | 2 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 13 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 26 ++-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 5 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 20 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 19 +-
.../datasources/Spark31NestedSchemaPruning.scala | 13 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 21 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 20 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 19 +-
.../datasources/Spark32NestedSchemaPruning.scala | 13 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 78 +------
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 6 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 20 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 19 +-
.../datasources/Spark33NestedSchemaPruning.scala | 7 +-
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 2 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 15 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 38 +++-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 41 +++-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 2 +-
.../hudi/sync/common/HoodieMetaSyncOperations.java | 7 +
.../apache/hudi/sync/common/HoodieSyncClient.java | 41 ++--
.../org/apache/hudi/utilities/UtilHelpers.java | 2 +-
.../ChainedSchemaPostProcessor.java | 3 +-
.../DeleteSupportSchemaPostProcessor.java | 3 +-
.../DropColumnSchemaPostProcessor.java | 3 +-
.../add/AddPrimitiveColumnSchemaPostProcessor.java | 97 ++++++++
.../add/BaseSchemaPostProcessorConfig.java | 53 +++++
.../hudi/utilities/TestSchemaPostProcessor.java | 56 ++++-
packaging/hudi-flink-bundle/pom.xml | 3 +-
packaging/hudi-spark-bundle/pom.xml | 3 +
pom.xml | 40 +++-
rfc/rfc-51/{points.jpg => code-paths.jpg} | Bin
rfc/rfc-51/{arch.jpg => logic-flows.jpg} | Bin
rfc/rfc-51/rfc-51.md | 217 +++++++++++-------
...io-definition.jpg => scenario-illustration.jpg} | Bin
200 files changed, 3657 insertions(+), 2269 deletions(-)
delete mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java
delete mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieSplashScreen.java
delete mode 100644
hudi-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml
copy
hudi-utilities/src/test/resources/delta-streamer-config/clusteringjob.properties
=> hudi-cli/src/main/resources/application.yml (91%)
create mode 100644 hudi-cli/src/main/resources/banner.txt
copy hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties =>
hudi-cli/src/main/resources/log4j2.properties (81%)
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/MockCommandLineInput.java
copy
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/{HoodieCLIIntegrationTestBase.java
=> ShellEvaluationResultUtil.java} (61%)
copy
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/{PartitionSortPartitionerWithRows.java
=> RowSpatialCurveSortPartitioner.java} (50%)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/SpatialCurveSortPartitionerBase.java
rename
{hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/internal
=>
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit}/BulkInsertDataInternalWriterHelper.java
(90%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala
(73%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala
(100%)
rename {hudi-spark-datasource/hudi-spark-common =>
hudi-client/hudi-spark-client}/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala
(99%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala
rename {hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table =>
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi}/functional/TestHoodieSparkMergeOnReadTableClustering.java
(87%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestQueryMergeOnReadOptimizedTable.scala
rename
hudi-cli/src/test/java/org/apache/hudi/cli/functional/CLIFunctionalTestSuite.java
=>
hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/encoders/DummyExpressionHolder.scala
(66%)
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/ChainedSchemaPostProcessor.java (93%)
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/DeleteSupportSchemaPostProcessor.java (95%)
rename hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/{ =>
postprocessor}/DropColumnSchemaPostProcessor.java (96%)
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/BaseSchemaPostProcessorConfig.java
rename rfc/rfc-51/{points.jpg => code-paths.jpg} (100%)
rename rfc/rfc-51/{arch.jpg => logic-flows.jpg} (100%)
rename rfc/rfc-51/{scenario-definition.jpg => scenario-illustration.jpg} (100%)
