This is an automated email from the ASF dual-hosted git repository.
aljoscha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 828faab [hotfix][javadocs] Fix typo
add 9e49d40 [hotfix][fs-connector] Remove unused state serializer from
FileWriter
add 15e1901 [hotfix][fs-connector] Fix the typo in the FileSinkITCase
add ff32471 [FLINK-20337] Let StatefulSinkWriterOperator load
StreamingFileSink's state
add cd6acf1 [FLINK-20337] Extend StatefulSinkWriterOperator Javadoc
add da1ad03 [FLINK-20337] Let FileSink restore state from
StreamingFileSink
add 2fe2be7 [refactor] Factor test data generator/resolver out of
BucketStateSerializerTest
add 0bcad04 [FLINK-20337] Add migration test for deserializing existing
StreamingFileSink state with FileSink serializer
add e302552 [refactor] Factor common testing code out of FileSinkITBase
add 794f340 [FLINK-20337] Add ITCase for migrating from StreamingFileSink
to FileSink
add 36d2945 [refactor] Move FileSink migration tests to
flink-connector-files
No new revisions were added by this update.
Summary of changes:
flink-connectors/flink-connector-files/pom.xml | 30 +++
.../apache/flink/connector/file/sink/FileSink.java | 14 +-
.../connector/file/sink/writer/FileWriter.java | 9 +-
.../file/sink/writer/FileWriterBucket.java | 12 +
.../file/sink/writer/FileWriterBucketState.java | 33 +++
.../writer/FileWriterBucketStateSerializer.java | 110 +++++++-
.../flink/connector/file/sink/FileSinkITBase.java | 93 +------
.../sink/StreamingExecutionFileSinkITCase.java | 2 +-
.../sink/utils/IntegerFileSinkTestDataUtils.java | 140 ++++++++++
.../file/sink/writer/FileSinkMigrationITCase.java | 287 +++++++++++++++++++++
...leWriterBucketStateSerializerMigrationTest.java | 285 ++++++++++++++++++++
.../FileWriterBucketStateSerializerTest.java | 4 +-
.../file/sink/writer/FileWriterBucketTest.java | 59 +++++
.../bucket-state-migration-test/empty-v1/snapshot | Bin
.../bucket-state-migration-test/empty-v2/snapshot | Bin
...inprogress.a88d5993-77bc-44ce-880b-9f2a43b59ab4 | 0
...inprogress.7c0f2bd7-3078-48e8-9af2-d8773fb949c5 | 0
...inprogress.6729a640-0585-4785-a652-89802950c663 | 0
...inprogress.b4bcb0e9-5c9e-45dd-8963-1b163343544d | 0
...inprogress.e1e9e48d-0db6-4dd7-8a4d-fb4ebe7ed8ac | 0
.../full-no-in-progress-v1-template/snapshot | Bin
...inprogress.1e22e72d-0ab2-493b-8b00-9edac4252cec | 0
...inprogress.3821f491-9fa1-48b2-b66b-655352a3c8ec | 0
...inprogress.0af18f41-d8f8-4a4e-a92e-de12851be20b | 0
...inprogress.a3d0f4d2-d6ad-4f83-ba62-ed4b1fa86db2 | 0
...inprogress.666acf3e-935c-4621-8171-f7c897496524 | 0
.../full-no-in-progress-v2-template/snapshot | Bin
...inprogress.8fec17e9-5d54-4fa9-aebb-70736fe03c82 | 0
...inprogress.0035b171-2759-403a-8d6c-4612b28a7a6c | 0
...inprogress.49da8048-af6b-4665-b4f6-b659cb38dc97 | 0
...inprogress.d13ec4e0-07b5-4f4e-9be8-9fb457cbcde9 | 0
...inprogress.123ac2c7-f92a-476a-a848-1369b93d82a7 | 0
...inprogress.32f5a28f-20e1-48da-9951-10e795133d64 | 0
.../full-v1-template/snapshot | Bin
...inprogress.9731063e-2b28-4701-8cc1-e706480b8022 | 0
...inprogress.1d423406-097a-4deb-bfde-d023d3477cd5 | 0
...inprogress.6a837aa3-4736-4098-a878-fdeffe227628 | 0
...inprogress.f121b73d-ac74-4fbd-b70d-f13e51c9132c | 0
...inprogress.a156884a-f090-4c3f-a271-0b63ab539c45 | 0
...inprogress.83c527c5-14dc-4d49-9f99-c915f2224f6a | 0
.../full-v2-template/snapshot | Bin
...inprogress.a70190d6-d080-43a8-b414-746b09d3a8a0 | 0
.../only-in-progress-v1/snapshot | Bin
...inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef | 0
.../only-in-progress-v2/snapshot | Bin
.../OutputStreamBasedPartFileWriter.java | 23 +-
.../operators/sink/StatefulSinkWriterOperator.java | 58 ++++-
.../sink/StatefulSinkWriterOperatorFactory.java | 18 +-
.../translators/SinkTransformationTranslator.java | 15 +-
.../sink/filesystem/BucketStateGenerator.java | 173 +++++++++++++
...Committer.java => BucketStatePathResolver.java} | 38 +--
.../sink/filesystem/BucketStateSerializerTest.java | 153 ++---------
.../sink/StatefulSinkWriterOperatorTest.java | 109 ++++++++
53 files changed, 1390 insertions(+), 275 deletions(-)
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
create mode 100644
flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileWriterBucketStateSerializerMigrationTest.java
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/empty-v1/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/empty-v2/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-0.inprogress.a88d5993-77bc-44ce-880b-9f2a43b59ab4
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-1.inprogress.7c0f2bd7-3078-48e8-9af2-d8773fb949c5
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-2.inprogress.6729a640-0585-4785-a652-89802950c663
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-3.inprogress.b4bcb0e9-5c9e-45dd-8963-1b163343544d
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/bucket/test-bucket/.part-0-4.inprogress.e1e9e48d-0db6-4dd7-8a4d-fb4ebe7ed8ac
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v1-template/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-0.inprogress.1e22e72d-0ab2-493b-8b00-9edac4252cec
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-1.inprogress.3821f491-9fa1-48b2-b66b-655352a3c8ec
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-2.inprogress.0af18f41-d8f8-4a4e-a92e-de12851be20b
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-3.inprogress.a3d0f4d2-d6ad-4f83-ba62-ed4b1fa86db2
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/bucket/test-bucket/.part-0-4.inprogress.666acf3e-935c-4621-8171-f7c897496524
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-no-in-progress-v2-template/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-0.inprogress.8fec17e9-5d54-4fa9-aebb-70736fe03c82
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-1.inprogress.0035b171-2759-403a-8d6c-4612b28a7a6c
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-2.inprogress.49da8048-af6b-4665-b4f6-b659cb38dc97
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-3.inprogress.d13ec4e0-07b5-4f4e-9be8-9fb457cbcde9
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-4.inprogress.123ac2c7-f92a-476a-a848-1369b93d82a7
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/bucket/test-bucket/.part-0-5.inprogress.32f5a28f-20e1-48da-9951-10e795133d64
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v1-template/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-0.inprogress.9731063e-2b28-4701-8cc1-e706480b8022
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-1.inprogress.1d423406-097a-4deb-bfde-d023d3477cd5
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-2.inprogress.6a837aa3-4736-4098-a878-fdeffe227628
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-3.inprogress.f121b73d-ac74-4fbd-b70d-f13e51c9132c
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-4.inprogress.a156884a-f090-4c3f-a271-0b63ab539c45
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/bucket/test-bucket/.part-0-5.inprogress.83c527c5-14dc-4d49-9f99-c915f2224f6a
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/full-v2-template/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/only-in-progress-v1/bucket/test-bucket/.part-0-0.inprogress.a70190d6-d080-43a8-b414-746b09d3a8a0
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/only-in-progress-v1/snapshot
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/only-in-progress-v2/bucket/test-bucket/.part-0-0.inprogress.10833090-dd8c-4e36-884d-bb9758a3a8ef
(100%)
copy {flink-streaming-java =>
flink-connectors/flink-connector-files}/src/test/resources/bucket-state-migration-test/only-in-progress-v2/snapshot
(100%)
create mode 100644
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BucketStateGenerator.java
copy
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/{utils/NoOpCommitter.java
=> BucketStatePathResolver.java} (52%)
