This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new e8325cc [FLINK-23487][s3] Update aws and presto-hive dependencies
e8325cc is described below
commit e8325ccc767740bcbe6505e2b2c64babb7714518
Author: Ingo Bürk <[email protected]>
AuthorDate: Mon Jul 26 08:55:53 2021 +0200
[FLINK-23487][s3] Update aws and presto-hive dependencies
Newer versions are required to make IRSA (IAM Role for Service Account)
work properly with S3. Since presto-hive now also directly depends on
aws-java-sdk-sts, we make sure to exclude it from presto-hive and use
our own version instead.
This fixes IRSA for s3-presto, though not yet s3-hadoop.
This closes #16717.
---
flink-filesystems/flink-s3-fs-base/pom.xml | 2 +-
.../src/main/resources/META-INF/NOTICE | 12 +-
flink-filesystems/flink-s3-fs-presto/pom.xml | 156 ++++++++++++++++-----
.../src/main/resources/META-INF/NOTICE | 36 +++--
.../META-INF/licenses/LICENSE-aopalliance | 1 +
.../fs/s3presto/PrestoS3FileSystemITCase.java | 5 +-
6 files changed, 157 insertions(+), 55 deletions(-)
diff --git a/flink-filesystems/flink-s3-fs-base/pom.xml
b/flink-filesystems/flink-s3-fs-base/pom.xml
index 9395781..f76f520 100644
--- a/flink-filesystems/flink-s3-fs-base/pom.xml
+++ b/flink-filesystems/flink-s3-fs-base/pom.xml
@@ -32,7 +32,7 @@ under the License.
<packaging>jar</packaging>
<properties>
- <fs.s3.aws.version>1.11.788</fs.s3.aws.version>
+ <fs.s3.aws.version>1.11.951</fs.s3.aws.version>
</properties>
<dependencies>
diff --git
a/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE
b/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE
index ca9c2e3..adea824 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE
+++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE
@@ -3,12 +3,12 @@ Copyright 2014-2021 The Apache Software Foundation
This project bundles the following dependencies under the Apache Software
License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
-- com.amazonaws:aws-java-sdk-core:1.11.788
-- com.amazonaws:aws-java-sdk-dynamodb:1.11.788
-- com.amazonaws:aws-java-sdk-kms:1.11.788
-- com.amazonaws:aws-java-sdk-s3:1.11.788
-- com.amazonaws:aws-java-sdk-sts:1.11.788
-- com.amazonaws:jmespath-java:1.11.788
+- com.amazonaws:aws-java-sdk-core:1.11.951
+- com.amazonaws:aws-java-sdk-dynamodb:1.11.951
+- com.amazonaws:aws-java-sdk-kms:1.11.951
+- com.amazonaws:aws-java-sdk-s3:1.11.951
+- com.amazonaws:aws-java-sdk-sts:1.11.951
+- com.amazonaws:jmespath-java:1.11.951
- com.fasterxml.jackson.core:jackson-annotations:2.12.1
- com.fasterxml.jackson.core:jackson-core:2.12.1
- com.fasterxml.jackson.core:jackson-databind:2.12.1
diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml
b/flink-filesystems/flink-s3-fs-presto/pom.xml
index e30c378..e8d4fef 100644
--- a/flink-filesystems/flink-s3-fs-presto/pom.xml
+++ b/flink-filesystems/flink-s3-fs-presto/pom.xml
@@ -33,7 +33,7 @@ under the License.
<packaging>jar</packaging>
<properties>
- <presto.version>0.187</presto.version>
+ <presto.version>0.257</presto.version>
</properties>
<dependencies>
@@ -59,7 +59,7 @@ under the License.
<artifactId>presto-hive</artifactId>
<version>${presto.version}</version>
<exclusions>
- <!-- use our AWS dependencies instead -->
+ <!-- Exclude AWS dependencies which we provide
ourselves (and in more recent versions). -->
<exclusion>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
@@ -68,12 +68,26 @@ under the License.
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.amazonaws</groupId>
+
<artifactId>aws-java-sdk-sts</artifactId>
+ </exclusion>
+
+ <!-- Unfortunately, presto-hive has a lot of
dependencies, many of which we don't need.
+ To try and keep our footprint small, we're
excluding dependencies on a best-effort
+ basis here.
+ The best way to do this is looking at
+
https://github.com/prestodb/presto/tree/${presto.version}/presto-hive/src/main/java/com/facebook/presto/hive/s3
+ and checking which dependencies are
actually used. When there is doubt, it is
+ probably easier to just bundle it. -->
- <!-- lot's of unneeded stuff for the S3 file
system -->
+ <!-- AWS -->
<exclusion>
- <groupId>com.facebook.hive</groupId>
- <artifactId>hive-dwrf</artifactId>
+ <groupId>com.amazonaws</groupId>
+
<artifactId>aws-java-sdk-glue</artifactId>
</exclusion>
+
+ <!-- Presto -->
<exclusion>
<groupId>com.facebook.presto.hive</groupId>
<artifactId>hive-apache</artifactId>
@@ -95,56 +109,96 @@ under the License.
<artifactId>presto-rcfile</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jdk14</artifactId>
+ <groupId>com.facebook.presto</groupId>
+ <artifactId>presto-parquet</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
-
<artifactId>log4j-over-slf4j</artifactId>
+ <groupId>com.facebook.presto</groupId>
+
<artifactId>presto-expressions</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
+ <groupId>com.facebook.presto</groupId>
+ <artifactId>presto-cache</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
+ <groupId>com.facebook.presto</groupId>
+
<artifactId>presto-memory-context</artifactId>
</exclusion>
+
+ <!-- Airlift -->
<exclusion>
<groupId>io.airlift</groupId>
+ <artifactId>aircompressor</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.facebook.airlift</groupId>
<artifactId>json</artifactId>
</exclusion>
<exclusion>
- <groupId>io.airlift</groupId>
+ <groupId>com.facebook.airlift</groupId>
<artifactId>bootstrap</artifactId>
</exclusion>
<exclusion>
- <groupId>io.airlift</groupId>
+ <groupId>com.facebook.airlift</groupId>
<artifactId>concurrent</artifactId>
</exclusion>
<exclusion>
- <groupId>io.airlift</groupId>
+ <groupId>com.facebook.airlift</groupId>
<artifactId>event</artifactId>
</exclusion>
<exclusion>
- <groupId>io.airlift</groupId>
+ <groupId>com.facebook.airlift</groupId>
<artifactId>http-client</artifactId>
</exclusion>
<exclusion>
- <groupId>io.airlift</groupId>
- <artifactId>aircompressor</artifactId>
+ <groupId>com.facebook.airlift</groupId>
+ <artifactId>log-manager</artifactId>
</exclusion>
+
+ <!-- Other -->
<exclusion>
- <groupId>io.airlift</groupId>
- <artifactId>log-manager</artifactId>
+ <groupId>cglib</groupId>
+ <artifactId>cglib-nodep</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.inject</groupId>
- <artifactId>javax.inject</artifactId>
+ <groupId>com.facebook.drift</groupId>
+ <artifactId>drift-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.facebook.hive</groupId>
+ <artifactId>hive-dwrf</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>util</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcsio</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>util-hadoop</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
</exclusion>
<exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
+ <groupId>com.google.errorprone</groupId>
+
<artifactId>error_prone_annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.j2objc</groupId>
+
<artifactId>j2objc-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+
<groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.inject.extensions</groupId>
@@ -155,28 +209,64 @@ under the License.
<artifactId>fastutil</artifactId>
</exclusion>
<exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
+ <groupId>javax.annotation</groupId>
+
<artifactId>javax.annotation-api</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.bval</groupId>
- <artifactId>bval-jsr</artifactId>
+ <groupId>javax.inject</groupId>
+ <artifactId>javax.inject</artifactId>
</exclusion>
<exclusion>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.apache.bval</groupId>
+ <artifactId>bval-jsr</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-hadoop-mr</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.mojo</groupId>
+
<artifactId>animal-sniffer-annotations</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.openjdk.jol</groupId>
<artifactId>jol-core</artifactId>
</exclusion>
<exclusion>
- <groupId>cglib</groupId>
- <artifactId>cglib-nodep</artifactId>
+ <groupId>org.roaringbitmap</groupId>
+ <artifactId>RoaringBitmap</artifactId>
</exclusion>
<exclusion>
-
<groupId>com.google.code.findbugs</groupId>
- <artifactId>annotations</artifactId>
+ <groupId>org.roaringbitmap</groupId>
+ <artifactId>shims</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+
<artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>
diff --git
a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE
b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE
index fdd7526..d133339 100644
--- a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE
+++ b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE
@@ -12,26 +12,31 @@ This project bundles the following dependencies under the
Apache Software Licens
- commons-io:commons-io:2.8.0
- commons-lang:commons-lang:2.6
- commons-logging:commons-logging:1.1.3
-- com.amazonaws:aws-java-sdk-core:1.11.788
-- com.amazonaws:aws-java-sdk-dynamodb:1.11.788
-- com.amazonaws:aws-java-sdk-kms:1.11.788
-- com.amazonaws:aws-java-sdk-s3:1.11.788
-- com.amazonaws:aws-java-sdk-sts:1.11.788
-- com.amazonaws:jmespath-java:1.11.788
-- com.facebook.presto:presto-hive:0.187
+- com.amazonaws:aws-java-sdk-core:1.11.951
+- com.amazonaws:aws-java-sdk-dynamodb:1.11.951
+- com.amazonaws:aws-java-sdk-kms:1.11.951
+- com.amazonaws:aws-java-sdk-s3:1.11.951
+- com.amazonaws:aws-java-sdk-sts:1.11.951
+- com.amazonaws:jmespath-java:1.11.951
+- com.facebook.presto:presto-common:0.257
+- com.facebook.presto:presto-hive:0.257
+- com.facebook.presto:presto-hive-common:0.257
+- com.facebook.presto:presto-hive-metastore:0.257
- com.facebook.presto.hadoop:hadoop-apache2:2.7.3-1
- com.fasterxml.jackson.core:jackson-annotations:2.12.1
- com.fasterxml.jackson.core:jackson-core:2.12.1
- com.fasterxml.jackson.core:jackson-databind:2.12.1
- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.1
- com.fasterxml.woodstox:woodstox-core:5.0.3
-- com.google.guava:guava:21.0
-- io.airlift:configuration:0.153
-- io.airlift:log:0.153
-- io.airlift:stats:0.153
-- io.airlift:units:1.0
-- io.airlift:slice:0.31
+- com.google.guava:guava:26.0-jre
+- com.google.inject:guice:4.2.2
+- com.facebook.airlift:configuration:0.201
+- com.facebook.airlift:log:0.201
+- com.facebook.airlift:stats:0.201
+- io.airlift:units:1.3
+- io.airlift:slice:0.38
- joda-time:joda-time:2.5
+- org.alluxio:alluxio-shaded-client:2.5.0-3
- org.apache.commons:commons-configuration2:2.1.1
- org.apache.commons:commons-lang3:3.3.2
- org.apache.hadoop:hadoop-annotations:3.1.0
@@ -63,3 +68,8 @@ This project bundles the following dependencies under BSD
License (https://opens
See bundled license files for details.
- org.codehaus.woodstox:stax2-api:3.1.4
(https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4)
+
+This project bundles the following dependencies under the Public Domain.
+See bundled license files for details.
+
+- aopalliance:aopalliance:1.0
diff --git
a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-aopalliance
b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-aopalliance
new file mode 100644
index 0000000..b56a63f
--- /dev/null
+++
b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/licenses/LICENSE-aopalliance
@@ -0,0 +1 @@
+all the source code provided by AOP Alliance is Public Domain.
diff --git
a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
index 82ba609..72d093a 100644
---
a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
+++
b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemITCase.java
@@ -24,6 +24,7 @@ import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.fs.hdfs.AbstractHadoopFileSystemITTest;
import org.apache.flink.testutils.s3.S3TestCredentials;
+import com.amazonaws.SdkClientException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -34,7 +35,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import static
com.facebook.presto.hive.s3.PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS;
+import static
com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
@@ -90,7 +91,7 @@ public class PrestoS3FileSystemITCase extends
AbstractHadoopFileSystemITTest {
try {
path.getFileSystem().exists(path);
fail("should fail with an exception");
- } catch (IOException ignored) {
+ } catch (SdkClientException ignored) {
}
}
