This is an automated email from the ASF dual-hosted git repository.
xtsong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 00eed85 [FLINK-24904][docs] Updated docs to reflect new KDS Sink and
deprecated old producer based on KPL, also updated metrics page to include KDS
Sink
add 0e4a666 [FLINK-11838][fs][gs] Create Google Storage file system with
RecoverableWriter support
No new revisions were added by this update.
Summary of changes:
.../java/org/apache/flink/core/fs/FileSystem.java | 1 +
flink-filesystems/flink-gs-fs-hadoop/pom.xml | 227 +++++++++++++++++
.../java/org/apache/flink/fs/gs/GSFileSystem.java | 60 +++++
.../apache/flink/fs/gs/GSFileSystemFactory.java | 109 ++++++++
.../apache/flink/fs/gs/GSFileSystemOptions.java | 92 +++++++
.../flink/fs/gs/storage/GSBlobIdentifier.java | 94 +++++++
.../apache/flink/fs/gs/storage/GSBlobStorage.java | 128 ++++++++++
.../flink/fs/gs/storage/GSBlobStorageImpl.java | 226 +++++++++++++++++
.../org/apache/flink/fs/gs/utils/BlobUtils.java | 128 ++++++++++
.../apache/flink/fs/gs/utils/ChecksumUtils.java | 50 ++++
.../flink/fs/gs/writer/GSChecksumWriteChannel.java | 116 +++++++++
.../flink/fs/gs/writer/GSCommitRecoverable.java | 99 ++++++++
.../gs/writer/GSCommitRecoverableSerializer.java | 153 ++++++++++++
.../gs/writer/GSRecoverableFsDataOutputStream.java | 244 ++++++++++++++++++
.../flink/fs/gs/writer/GSRecoverableWriter.java | 116 +++++++++
.../fs/gs/writer/GSRecoverableWriterCommitter.java | 232 +++++++++++++++++
.../flink/fs/gs/writer/GSResumeRecoverable.java | 73 ++++++
.../gs/writer/GSResumeRecoverableSerializer.java | 122 +++++++++
.../src/main/resources/META-INF/NOTICE | 71 ++++++
.../resources/META-INF/licenses/LICENSE.api-common | 27 ++
.../main/resources/META-INF/licenses/LICENSE.gax | 27 ++
.../META-INF/licenses/LICENSE.gax-httpjson | 27 ++
.../LICENSE.google-auth-library-credentials | 0
.../LICENSE.google-auth-library-oauth2-http | 0
.../META-INF/licenses/LICENSE.protobuf-java | 0
.../META-INF/licenses/LICENSE.protobuf-java-util | 0
.../resources/META-INF/licenses/LICENSE.threetenbp | 29 +++
.../org.apache.flink.core.fs.FileSystemFactory | 16 ++
.../flink/fs/gs/GSFileSystemScenarioTest.java | 275 +++++++++++++++++++++
.../java/org/apache/flink/fs/gs/TestUtils.java | 25 ++
.../flink/fs/gs/storage/MockBlobStorage.java | 205 +++++++++++++++
.../apache/flink/fs/gs/utils/BlobUtilsTest.java | 125 ++++++++++
.../flink/fs/gs/utils/ChecksumUtilsTest.java | 33 +++
.../fs/gs/writer/GSChecksumWriteChannelTest.java | 175 +++++++++++++
.../writer/GSCommitRecoverableSerializerTest.java | 90 +++++++
.../fs/gs/writer/GSCommitRecoverableTest.java | 136 ++++++++++
.../GSRecoverableFsDataOutputStreamTest.java | 268 ++++++++++++++++++++
.../writer/GSRecoverableWriterCommitterTest.java | 212 ++++++++++++++++
.../fs/gs/writer/GSRecoverableWriterTest.java | 170 +++++++++++++
.../writer/GSResumeRecoverableSerializerTest.java | 102 ++++++++
.../fs/gs/writer/GSResumeRecoverableTest.java | 116 +++++++++
.../flink/runtime/fs/hdfs/HadoopFileSystem.java | 4 +-
flink-filesystems/pom.xml | 1 +
43 files changed, 4403 insertions(+), 1 deletion(-)
create mode 100644 flink-filesystems/flink-gs-fs-hadoop/pom.xml
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/GSFileSystem.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/GSFileSystemFactory.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/GSFileSystemOptions.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/storage/GSBlobIdentifier.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/storage/GSBlobStorage.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/storage/GSBlobStorageImpl.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/utils/BlobUtils.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/utils/ChecksumUtils.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSChecksumWriteChannel.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSCommitRecoverable.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSCommitRecoverableSerializer.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSRecoverableFsDataOutputStream.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSRecoverableWriter.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSRecoverableWriterCommitter.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSResumeRecoverable.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/java/org/apache/flink/fs/gs/writer/GSResumeRecoverableSerializer.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/NOTICE
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.api-common
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.gax
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.gax-httpjson
copy {flink-python =>
flink-filesystems/flink-gs-fs-hadoop}/src/main/resources/META-INF/licenses/LICENSE.google-auth-library-credentials
(100%)
copy
flink-python/src/main/resources/META-INF/licenses/LICENSE.google-auth-library-credentials
=>
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.google-auth-library-oauth2-http
(100%)
copy flink-python/src/main/resources/META-INF/licenses/LICENSE.protobuf =>
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.protobuf-java
(100%)
copy flink-python/src/main/resources/META-INF/licenses/LICENSE.protobuf =>
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.protobuf-java-util
(100%)
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/licenses/LICENSE.threetenbp
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/GSFileSystemScenarioTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/TestUtils.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/storage/MockBlobStorage.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/utils/BlobUtilsTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/utils/ChecksumUtilsTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSChecksumWriteChannelTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSCommitRecoverableSerializerTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSCommitRecoverableTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSRecoverableFsDataOutputStreamTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSRecoverableWriterCommitterTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSRecoverableWriterTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSResumeRecoverableSerializerTest.java
create mode 100644
flink-filesystems/flink-gs-fs-hadoop/src/test/java/org/apache/flink/fs/gs/writer/GSResumeRecoverableTest.java
