This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
from a4627428351 [HUDI-7559] [1/n] Fix
RecordLevelIndexSupport::filterQueryWithRecordKey (#10947)
new 4271e01e2e1 [MINOR] Optimize print write error msg in
StreamWriteOperatorCoordinator#doCommit (#10809)
new 07f27155115 [HUDI-7556] Fixing false positive validation with MDT
validator (#10986)
new 1a8949cc49f [HUDI-7583] Read log block header only for the schema and
instant time (#10984)
new 51d5346d3db [HUDI-7597] Add logs of Kafka offsets when the checkpoint
is out of bound (#10987)
new 2671b3a6828 [MINOR] Fix BUG: HoodieLogFormatWriter: unable to close
output stream for log file HoodieLogFile{xxx} (#10989)
new ea01c0ff194 [HUDI-7600] Shutdown ExecutorService when
HiveMetastoreBasedLockProvider is closed (#10993)
new 02757525993 [HUDI-7391] HoodieMetadataMetrics should use Metrics
instance for metrics registry (#10635)
new 35b0a051345 [HUDI-6441] Passing custom Headers with Hudi Callback URL
(#10970)
new f82023aecb5 [HUDI-7605] Allow merger strategy to be set in spark sql
writer (#10999)
new 4a993dae5df [HUDI-7290] Don't assume ReplaceCommits are always
Clustering (#10479)
new bf2ed196cd0 [HUDI-7601] Add heartbeat mechanism to refresh lock
(#10994)
new 3f5c86bdd90 [HUDI-7378] Fix Spark SQL DML with custom key generator
(#10615)
new 7b8846594f5 [HUDI-7616] Avoid multiple cleaner plans and deprecate
hoodie.clean.allow.multiple (#11013)
new e2777a96f1b [HUDI-7606] Unpersist RDDs after table services, mainly
compaction and clustering (#11000)
new 6c298a0f0f9 [HUDI-7615] Mark a few write configs with the correct
sinceVersion (#11012)
new c357b4d18ca [HUDI-7584] Always read log block lazily and remove
readBlockLazily argument (#11015)
new ff5c31ef781 [HUDI-7619] Removed code duplicates in
HoodieTableMetadataUtil (#11022)
new d1b4872e319 [HUDI-6762] Removed usages of
MetadataRecordsGenerationParams (#10962)
new 65a0a635126 [MINOR] Remove redundant lines in StreamSync and
TestStreamSyncUnitTests (#11027)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/cli/commands/HoodieLogFileCommand.java | 3 -
.../cli/commands/TestHoodieLogFileCommand.java | 3 -
hudi-client/hudi-client-common/pom.xml | 46 --
.../http/HoodieWriteCommitHttpCallbackClient.java | 46 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 22 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 2 +-
.../lock/metrics/HoodieLockMetrics.java | 2 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 4 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 13 +
.../config/HoodieWriteCommitCallbackConfig.java | 15 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 98 ++--
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 1 -
.../metadata/HoodieBackedTableMetadataWriter.java | 118 ++---
.../hudi/metadata/HoodieMetadataWriteUtils.java | 9 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 2 +-
.../cloudwatch/CloudWatchMetricsReporter.java | 29 +-
.../hudi/table/action/compact/HoodieCompactor.java | 1 -
.../table/action/index/RunIndexActionExecutor.java | 3 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 2 +
.../client/http/TestCallbackHttpClient.java | 202 ++++++++
.../hudi/callback/http/TestCallbackHttpClient.java | 143 ------
.../hudi/metrics/TestHoodieConsoleMetrics.java | 16 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 22 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 19 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 17 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 20 +-
.../cloudwatch/TestCloudWatchMetricsReporter.java | 27 +-
.../datadog/TestDatadogMetricsReporter.java | 60 ++-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 54 +-
.../metrics/prometheus/TestPrometheusReporter.java | 19 +-
.../prometheus/TestPushGateWayReporter.java | 52 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 4 +-
.../run/strategy/JavaExecutionStrategy.java | 1 -
.../JavaHoodieBackedTableMetadataWriter.java | 4 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 21 +-
.../hudi/client/SparkRDDTableServiceClient.java | 6 +
.../apache/hudi/client/SparkRDDWriteClient.java | 21 +-
.../MultipleSparkJobExecutionStrategy.java | 1 -
.../hudi/client/utils/SparkReleaseResources.java | 64 +++
.../factory/HoodieSparkKeyGeneratorFactory.java | 4 +
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 16 +-
.../functional/TestHoodieBackedMetadata.java | 18 +-
.../java/org/apache/hudi/table/TestCleaner.java | 6 +-
hudi-common/pom.xml | 47 ++
.../hudi/common/config/LockConfiguration.java | 3 +
.../hudi/common/table/TableSchemaResolver.java | 24 +-
.../table/log/AbstractHoodieLogRecordReader.java | 65 +--
.../table/log/HoodieCDCLogRecordIterator.java | 3 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 69 ++-
.../hudi/common/table/log/HoodieLogFormat.java | 13 +-
.../common/table/log/HoodieLogFormatReader.java | 14 +-
.../common/table/log/HoodieLogFormatWriter.java | 2 +-
.../table/log/HoodieMergedLogRecordScanner.java | 27 +-
.../table/log/HoodieUnMergedLogRecordScanner.java | 12 +-
.../hudi/common/table/log/LogReaderUtils.java | 2 +-
.../table/timeline/HoodieDefaultTimeline.java | 31 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 11 +
.../table/view/AbstractTableFileSystemView.java | 5 +-
.../metrics/HoodieMetricsCloudWatchConfig.java | 0
.../hudi/config/metrics/HoodieMetricsConfig.java | 201 ++++++++
.../config/metrics/HoodieMetricsDatadogConfig.java | 0
.../metrics/HoodieMetricsGraphiteConfig.java | 0
.../config/metrics/HoodieMetricsJmxConfig.java | 0
.../hudi/config/metrics/HoodieMetricsM3Config.java | 0
.../metrics/HoodieMetricsPrometheusConfig.java | 0
.../hudi/keygen/constant/KeyGeneratorOptions.java | 1 +
.../apache/hudi/metadata/BaseTableMetadata.java | 4 +-
.../metadata/HoodieMetadataLogRecordReader.java | 1 -
.../hudi/metadata/HoodieMetadataMetrics.java | 21 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 365 ++++++-------
.../metadata/MetadataRecordsGenerationParams.java | 89 ----
.../hudi/metrics/ConsoleMetricsReporter.java | 0
.../java/org/apache/hudi/metrics/HoodieGauge.java | 0
.../hudi/metrics/InMemoryMetricsReporter.java | 0
.../apache/hudi/metrics/JmxMetricsReporter.java | 4 +-
.../org/apache/hudi/metrics/JmxReporterServer.java | 0
.../java/org/apache/hudi/metrics/MetricUtils.java | 0
.../main/java/org/apache/hudi/metrics/Metrics.java | 43 +-
.../hudi/metrics/MetricsGraphiteReporter.java | 16 +-
.../org/apache/hudi/metrics/MetricsReporter.java | 0
.../hudi/metrics/MetricsReporterFactory.java | 27 +-
.../apache/hudi/metrics/MetricsReporterType.java | 0
.../custom/CustomizableMetricsReporter.java | 0
.../hudi/metrics/datadog/DatadogHttpClient.java | 0
.../metrics/datadog/DatadogMetricsReporter.java | 4 +-
.../hudi/metrics/datadog/DatadogReporter.java | 0
.../apache/hudi/metrics/m3/M3MetricsReporter.java | 16 +-
.../hudi/metrics/m3/M3ScopeReporterAdaptor.java | 0
.../metrics/prometheus/PrometheusReporter.java | 10 +-
.../prometheus/PushGatewayMetricsReporter.java | 18 +-
.../metrics/prometheus/PushGatewayReporter.java | 0
.../AbstractUserDefinedMetricsReporter.java | 0
.../common/functional/TestHoodieLogFormat.java | 130 ++---
.../hudi/common/table/TestTableSchemaResolver.java | 56 ++
.../table/view/TestHoodieTableFileSystemView.java | 30 +-
.../examples/quickstart/TestQuickstartData.java | 1 -
.../hudi/sink/StreamWriteOperatorCoordinator.java | 15 +-
.../hudi/sink/clustering/ClusteringOperator.java | 1 -
.../clustering/ClusteringPlanSourceFunction.java | 2 +-
.../org/apache/hudi/table/format/FormatUtils.java | 6 -
.../java/org/apache/hudi/util/ClusteringUtil.java | 2 +-
.../test/java/org/apache/hudi/utils/TestData.java | 1 -
.../realtime/HoodieMergeOnReadSnapshotReader.java | 3 -
.../realtime/RealtimeCompactedRecordReader.java | 1 -
.../realtime/RealtimeUnmergedRecordReader.java | 1 -
.../reader/DFSHoodieDatasetInputReader.java | 1 -
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 1 +
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 20 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 4 -
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 60 ++-
.../spark/sql/hudi/TestProvidesHoodieConfig.scala | 79 +++
.../hudi/command/MergeIntoHoodieTableCommand.scala | 5 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 1 -
.../apache/hudi/functional/TestMORDataSource.scala | 20 +
.../TestSparkSqlWithCustomKeyGenerator.scala | 571 +++++++++++++++++++++
.../hudi/hive/transaction/lock/Heartbeat.java | 25 +-
.../lock/HiveMetastoreBasedLockProvider.java | 24 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 12 +-
.../utilities/HoodieMetadataTableValidator.java | 218 ++++----
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 8 +-
.../ingestion/HoodieIngestionMetrics.java | 7 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 29 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 5 +
.../apache/hudi/utilities/streamer/StreamSync.java | 6 +-
.../TestHoodieMetadataTableValidator.java | 131 ++++-
.../utilities/sources/BaseTestKafkaSource.java | 16 +-
.../streamer/TestStreamSyncUnitTests.java | 20 -
128 files changed, 2614 insertions(+), 1284 deletions(-)
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/client/http/TestCallbackHttpClient.java
delete mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkReleaseResources.java
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsCloudWatchConfig.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
(60%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsM3Config.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java
(100%)
delete mode 100644
hudi-common/src/main/java/org/apache/hudi/metadata/MetadataRecordsGenerationParams.java
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/HoodieGauge.java (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java (96%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/MetricUtils.java (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/Metrics.java (80%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java
(84%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsReporter.java (100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
(73%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/datadog/DatadogHttpClient.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/datadog/DatadogMetricsReporter.java
(95%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/datadog/DatadogReporter.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java
(88%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/prometheus/PrometheusReporter.java
(92%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayMetricsReporter.java
(79%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayReporter.java
(100%)
rename {hudi-client/hudi-client-common =>
hudi-common}/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java
(100%)
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FiveToFourDowngradeHandler.java
=>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java
(59%)
