This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from 2933249 [HUDI-2298] The HoodieMergedLogRecordScanner should set up
the operation of the chosen record (#3456)
add 4783176 [HUDI-1138] Add timeline-server-based marker file strategy
for improving marker-related latency (#3233)
No new revisions were added by this update.
Summary of changes:
.../hudi/client/AbstractHoodieWriteClient.java | 5 +-
.../embedded/EmbeddedTimelineServerHelper.java | 6 +-
.../client/embedded/EmbeddedTimelineService.java | 54 +--
.../org/apache/hudi/config/HoodieWriteConfig.java | 53 +++
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 7 +-
.../hudi/table/HoodieTimelineArchiveLog.java | 11 +-
.../rollback/BaseRollbackActionExecutor.java | 8 +-
.../DirectWriteMarkers.java} | 118 ++-----
.../org/apache/hudi/table/marker/MarkerType.java | 9 +-
.../marker/TimelineServerBasedWriteMarkers.java | 179 ++++++++++
.../org/apache/hudi/table/marker/WriteMarkers.java | 186 ++++++++++
.../hudi/table/marker/WriteMarkersFactory.java | 58 ++++
.../apache/hudi/client/HoodieFlinkWriteClient.java | 4 +-
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 10 +-
.../java/org/apache/hudi/io/FlinkCreateHandle.java | 7 +-
.../apache/hudi/io/FlinkMergeAndReplaceHandle.java | 7 +-
.../java/org/apache/hudi/io/FlinkMergeHandle.java | 7 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 7 +-
.../rollback/FlinkMarkerBasedRollbackStrategy.java | 14 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 9 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 38 ++-
.../rollback/JavaMarkerBasedRollbackStrategy.java | 15 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 5 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 6 +-
.../rollback/SparkMarkerBasedRollbackStrategy.java | 14 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 9 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 38 ++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 9 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 5 +-
.../TestMergeOnReadRollbackActionExecutor.java | 4 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 74 ++++
.../TestTimelineServerBasedWriteMarkers.java | 141 ++++++++
.../TestWriteMarkersBase.java} | 83 ++---
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 25 +-
.../hudi/common/table/marker/MarkerOperation.java | 41 +++
.../org/apache/hudi/common/util/FileIOUtils.java | 22 +-
.../hudi/exception/HoodieRemoteException.java | 3 +
.../hudi/timeline/service/RequestHandler.java | 123 +++++--
.../hudi/timeline/service/TimelineService.java | 174 ++++++++--
.../timeline/service/handlers/BaseFileHandler.java | 7 +-
.../service/handlers/FileSliceHandler.java | 8 +-
.../hudi/timeline/service/handlers/Handler.java | 8 +-
.../timeline/service/handlers/MarkerHandler.java | 198 +++++++++++
.../timeline/service/handlers/TimelineHandler.java | 7 +-
.../marker/BatchedMarkerCreationContext.java | 57 ++++
.../marker/BatchedMarkerCreationRunnable.java | 51 +++
.../marker/MarkerCreationDispatchingRunnable.java | 90 +++++
.../handlers/marker/MarkerCreationFuture.java | 69 ++++
.../service/handlers/marker/MarkerDirState.java | 375 +++++++++++++++++++++
.../TestRemoteHoodieTableFileSystemView.java | 5 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 10 +-
52 files changed, 2122 insertions(+), 357 deletions(-)
rename
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/{MarkerFiles.java
=> marker/DirectWriteMarkers.java} (61%)
copy
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCleaningPolicy.java
=>
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/MarkerType.java
(83%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/TimelineServerBasedWriteMarkers.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkers.java
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/WriteMarkersFactory.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestDirectWriteMarkers.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java
rename
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/{TestMarkerFiles.java
=> marker/TestWriteMarkersBase.java} (55%)
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/table/marker/MarkerOperation.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/MarkerHandler.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/BatchedMarkerCreationContext.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/BatchedMarkerCreationRunnable.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerCreationDispatchingRunnable.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerCreationFuture.java
create mode 100644
hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java
