This is an automated email from the ASF dual-hosted git repository.
dannycranmer pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 8a9a08b [FLINK-25041][e2e] Improve error handling for missing
environment variables in E2E tests
new 0e908be [FLINK-24228][connectors/firehose] Amazon Kinesis Data
Firehose sink based on the Async Sink Base implemented: - Refactored
AWSUnifiedSinksUtil into a class that caters for Kinesis and Firehose -
Extracted commonalities between KDS & KDF sinks into flink-connector-aws-base
- Implemented integration test based on Localstack container - Changing
host/container ports to be different, changing HTTP1.1 to being the default,
localstack issue fixed - Added docs page, chang [...]
new edcd0b0 [FLINK-24228][connectors/firehose] Added documentation (&.zh
docs) for the new Firehose sink.
new 9050fd5 [FLINK-24228][connectors/firehose] Allows end users to supply
a serialization schema rather than an ElementConverter, thereby encapsulating
the Firehose `Record` from the user, verifying stream objects in
KinesisFirehoseITCase
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../docs/connectors/datastream/firehose.md | 161 +++++++++++++++++++
.../content/docs/connectors/datastream/firehose.md | 161 +++++++++++++++++++
flink-connectors/flink-connector-aws-base/pom.xml | 21 +++
.../connector/aws/util/AWSAsyncSinkUtil.java} | 107 +++++++------
.../aws/testutils/AWSServicesTestUtils.java | 173 +++++++++++++++++++++
.../aws/testutils/LocalstackContainer.java | 78 ++++++++++
.../connector/aws/util/AWSAsyncSinkUtilTest.java} | 150 +++++++++++-------
.../pom.xml | 12 --
.../KinesisDataStreamsConfigConstants.java} | 9 +-
.../kinesis/sink/KinesisDataStreamsSinkWriter.java | 10 +-
.../src/test/resources/log4j2-test.properties | 2 +-
.../pom.xml | 43 +++--
.../sink/KinesisFirehoseConfigConstants.java} | 17 +-
.../firehose/sink/KinesisFirehoseException.java | 54 +++++++
.../firehose/sink/KinesisFirehoseSink.java | 115 ++++++++++++++
.../firehose/sink/KinesisFirehoseSinkBuilder.java | 162 +++++++++++++++++++
.../sink/KinesisFirehoseSinkElementConverter.java | 74 +++++++++
.../firehose/sink/KinesisFirehoseSinkWriter.java} | 119 +++++++-------
.../src/main/resources/log4j2.properties | 0
.../sink/KinesisFirehoseSinkBuilderTest.java | 68 ++++++++
.../KinesisFirehoseSinkElementConverterTest.java | 54 +++++++
.../firehose/sink/KinesisFirehoseSinkITCase.java | 131 ++++++++++++++++
.../firehose/sink/KinesisFirehoseSinkTest.java | 76 +++++++++
.../sink/KinesisFirehoseSinkWriterTest.java | 107 +++++++++++++
.../sink/testutils/KinesisFirehoseTestUtils.java | 73 +++++++++
.../src/test}/resources/log4j2-test.properties | 0
flink-connectors/flink-connector-base/pom.xml | 16 ++
.../base/sink/writer/AsyncSinkWriterTest.java | 120 +-------------
.../base/sink/writer/TestSinkInitContext.java | 139 +++++++++++++++++
.../connectors/kinesis/proxy/KinesisProxyV2.java | 4 +-
.../kinesis/proxy/KinesisProxyV2Factory.java | 18 ++-
.../streaming/connectors/kinesis/util/AWSUtil.java | 11 +-
.../connectors/kinesis/util/AwsV2Util.java | 8 +
.../connectors/kinesis/util/AwsV2UtilTest.java | 34 ++++
flink-connectors/pom.xml | 1 +
.../org/apache/flink/util/DockerImageVersions.java | 2 +
tools/ci/stage.sh | 1 +
37 files changed, 1977 insertions(+), 354 deletions(-)
create mode 100644 docs/content.zh/docs/connectors/datastream/firehose.md
create mode 100644 docs/content/docs/connectors/datastream/firehose.md
rename
flink-connectors/{flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/util/AWSKinesisDataStreamsUtil.java
=>
flink-connector-aws-base/src/main/java/org/apache/flink/connector/aws/util/AWSAsyncSinkUtil.java}
(61%)
create mode 100644
flink-connectors/flink-connector-aws-base/src/test/java/org/apache/flink/connector/aws/testutils/AWSServicesTestUtils.java
create mode 100644
flink-connectors/flink-connector-aws-base/src/test/java/org/apache/flink/connector/aws/testutils/LocalstackContainer.java
rename
flink-connectors/{flink-connector-aws-kinesis-data-streams/src/test/java/org/apache/flink/connector/kinesis/util/AWSKinesisDataStreamsUtilTest.java
=>
flink-connector-aws-base/src/test/java/org/apache/flink/connector/aws/util/AWSAsyncSinkUtilTest.java}
(62%)
copy
flink-connectors/flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/{config/AWSKinesisDataStreamsConfigConstants.java
=> sink/KinesisDataStreamsConfigConstants.java} (80%)
copy flink-connectors/{flink-connector-aws-kinesis-data-streams =>
flink-connector-aws-kinesis-firehose}/pom.xml (83%)
rename
flink-connectors/{flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/config/AWSKinesisDataStreamsConfigConstants.java
=>
flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseConfigConstants.java}
(62%)
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseException.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSink.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkBuilder.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkElementConverter.java
copy
flink-connectors/{flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/sink/KinesisDataStreamsSinkWriter.java
=>
flink-connector-aws-kinesis-firehose/src/main/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkWriter.java}
(54%)
copy flink-connectors/{flink-connector-base =>
flink-connector-aws-kinesis-firehose}/src/main/resources/log4j2.properties
(100%)
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkBuilderTest.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkElementConverterTest.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkITCase.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkTest.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/KinesisFirehoseSinkWriterTest.java
create mode 100644
flink-connectors/flink-connector-aws-kinesis-firehose/src/test/java/org/apache/flink/connector/firehose/sink/testutils/KinesisFirehoseTestUtils.java
copy {flink-test-utils-parent/flink-connector-test-utils/src/main =>
flink-connectors/flink-connector-aws-kinesis-firehose/src/test}/resources/log4j2-test.properties
(100%)
create mode 100644
flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/sink/writer/TestSinkInitContext.java
