This is an automated email from the ASF dual-hosted git repository. arvid pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit f85786146f72e66280fef1c17884b18c07b7142c Author: Srinivasulu Punuru <[email protected]> AuthorDate: Thu Aug 5 10:27:07 2021 -0700 [FLINK-18562][fs] Upgrade hadoop to 3.3.1 - Needed for ABFS support --- .../src/main/resources/META-INF/NOTICE | 15 ++++++--- .../src/main/resources/META-INF/NOTICE | 35 ++++++++++++++++---- .../src/main/resources/META-INF/NOTICE | 2 +- .../flink/fs/s3hadoop/HadoopS3AccessHelper.java | 26 +++++++++++++-- .../src/main/resources/META-INF/NOTICE | 38 +++++++++++++++++----- .../src/main/resources/META-INF/NOTICE | 28 ++++++++++++---- flink-filesystems/pom.xml | 2 +- 7 files changed, 115 insertions(+), 31 deletions(-) diff --git a/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE index 9d48770..0631b50 100644 --- a/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-azure-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -9,13 +9,20 @@ This project bundles the following dependencies under the Apache Software Licens - com.fasterxml.jackson.core:jackson-annotations:2.12.1 - com.fasterxml.jackson.core:jackson-core:2.12.1 - com.fasterxml.jackson.core:jackson-databind:2.12.1 -- com.google.guava:guava:11.0.2 -- com.microsoft.azure:azure-keyvault-core:0.8.0 -- com.microsoft.azure:azure-storage:5.4.0 +- com.google.guava:guava:20.0 - commons-codec:commons-codec:1.13 - commons-logging:commons-logging:1.1.3 -- org.apache.hadoop:hadoop-azure:3.1.0 +- org.apache.hadoop:hadoop-azure:3.3.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 - org.apache.httpcomponents:httpclient:4.5.13 - org.apache.httpcomponents:httpcore:4.4.14 +- org.codehaus.jackson:jackson-mapper-asl:1.9.13 +- org.codehaus.jackson:jackson-core-asl:1.9.13 - org.eclipse.jetty:jetty-util:9.3.24.v20180605 - org.eclipse.jetty:jetty-util-ajax:9.3.24.v20180605 +- org.wildfly.openssl:wildfly-openssl:1.0.7.Final + +This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT) + +- com.microsoft.azure:azure-keyvault-core:1.0.0 +- com.microsoft.azure:azure-storage:7.0.1 diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE index 2d887f2..3ccf2ac 100644 --- a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/META-INF/NOTICE @@ -6,22 +6,43 @@ The Apache Software Foundation (http://www.apache.org/). This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.hadoop:hadoop-annotations:3.3.1 +- org.apache.hadoop:hadoop-auth:3.3.1 +- org.apache.hadoop:hadoop-common:3.3.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1 - org.apache.htrace:htrace-core4:4.1.0-incubating +- org.apache.kerby:kerb-core:1.0.1 +- org.apache.kerby:kerby-pkix:1.0.1 +- org.apache.kerby:kerby-asn1:1.0.1 +- org.apache.kerby:kerby-util:1.0.1 - org.apache.commons:commons-configuration2:2.1.1 - org.apache.commons:commons-lang3:3.3.2 +- org.apache.commons:commons-text:1.4 +- org.xerial.snappy:snappy-java:1.1.8.3 - commons-lang:commons-lang:2.6 - commons-collections:commons-collections:3.2.2 - commons-io:commons-io:2.8.0 - commons-logging:commons-logging:1.1.3 - commons-beanutils:commons-beanutils:1.9.4 -- com.google.guava:guava:11.0.2 +- com.google.guava:failureaccess:1.0 +- com.google.guava:guava:27.0-jre +- com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +- com.google.j2objc:j2objc-annotations:1.1 - com.fasterxml.jackson.core:jackson-annotations:2.12.1 - com.fasterxml.jackson.core:jackson-core:2.12.1 - com.fasterxml.jackson.core:jackson-databind:2.12.1 -- com.fasterxml.woodstox:woodstox-core:5.0.3 +- com.fasterxml.woodstox:woodstox-core:5.3.0 + +This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT) + +- org.checkerframework:checker-qual:2.5.2 +- org.codehaus.mojo:animal-sniffer-annotations:1.17 + +This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause). +See bundled license files for details. + +- dnsjava:dnsjava:2.1.7 This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). See bundled license files for details. @@ -31,9 +52,9 @@ See bundled license files for details. This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). See bundled license files for details. -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) +- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1) -This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: +This project bundles org.apache.hadoop:*:3.3.1 from which it inherits the following notices: The Apache Hadoop project contains subcomponents with separate copyright notices and license terms. Your use of the source code for the these diff --git a/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/NOTICE index c81d333..3148dda 100644 --- a/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-oss-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -13,7 +13,7 @@ This project bundles the following dependencies under the Apache Software Licens - com.aliyun:aliyun-java-sdk-sts:3.0.0 - commons-codec:commons-codec:1.13 - commons-logging:commons-logging:1.1.3 -- org.apache.hadoop:hadoop-aliyun:3.1.0 +- org.apache.hadoop:hadoop-aliyun:3.3.1 - org.apache.httpcomponents:httpclient:4.5.13 - org.apache.httpcomponents:httpcore:4.4.14 - org.codehaus.jettison:jettison:1.1 diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java index efc8ecd..a4f690b 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java +++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/HadoopS3AccessHelper.java @@ -30,10 +30,13 @@ import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3AInstrumentation; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.WriteOperationHelper; +import org.apache.hadoop.fs.s3a.statistics.impl.BondedS3AStatisticsContext; import java.io.File; import java.io.FileOutputStream; @@ -52,8 +55,22 @@ public class HadoopS3AccessHelper implements S3AccessHelper { private final InternalWriteOperationHelper s3accessHelper; public HadoopS3AccessHelper(S3AFileSystem s3a, Configuration conf) { + FileSystem.Statistics statistics = new FileSystem.Statistics(s3a.getScheme()); + BondedS3AStatisticsContext statisticsContext = + new BondedS3AStatisticsContext( + new BondedS3AStatisticsContext.S3AFSStatisticsSource() { + public S3AInstrumentation getInstrumentation() { + return s3a.getInstrumentation(); + } + + public FileSystem.Statistics getInstanceStatistics() { + return statistics; + } + }); + this.s3accessHelper = - new InternalWriteOperationHelper(checkNotNull(s3a), checkNotNull(conf)); + new InternalWriteOperationHelper( + checkNotNull(s3a), checkNotNull(conf), statisticsContext); this.s3a = s3a; } @@ -144,8 +161,11 @@ public class HadoopS3AccessHelper implements S3AccessHelper { */ private static final class InternalWriteOperationHelper extends WriteOperationHelper { - InternalWriteOperationHelper(S3AFileSystem owner, Configuration conf) { - super(owner, conf); + InternalWriteOperationHelper( + S3AFileSystem owner, + Configuration conf, + BondedS3AStatisticsContext statisticsContext) { + super(owner, conf, statisticsContext); } } } diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE index adea824..4c7f1b1 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/resources/META-INF/NOTICE @@ -13,8 +13,11 @@ This project bundles the following dependencies under the Apache Software Licens - com.fasterxml.jackson.core:jackson-core:2.12.1 - com.fasterxml.jackson.core:jackson-databind:2.12.1 - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.1 -- com.fasterxml.woodstox:woodstox-core:5.0.3 -- com.google.guava:guava:11.0.2 +- com.fasterxml.woodstox:woodstox-core:5.3.0 +- com.google.guava:failureaccess:1.0 +- com.google.guava:guava:27.0-jre +- com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +- com.google.j2objc:j2objc-annotations:1.1 - commons-beanutils:commons-beanutils:1.9.4 - commons-codec:commons-codec:1.13 - commons-collections:commons-collections:3.2.2 @@ -24,15 +27,34 @@ This project bundles the following dependencies under the Apache Software Licens - joda-time:joda-time:2.5 - org.apache.commons:commons-configuration2:2.1.1 - org.apache.commons:commons-lang3:3.3.2 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-aws:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.commons:commons-text:1.4 +- org.apache.hadoop:hadoop-auth:3.3.1 +- org.apache.hadoop:hadoop-annotations:3.3.1 +- org.apache.hadoop:hadoop-aws:3.3.1 +- org.apache.hadoop:hadoop-common:3.3.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1 - org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.httpcomponents:httpcore:4.4.14 - org.apache.httpcomponents:httpclient:4.5.13 +- org.apache.httpcomponents:httpcore:4.4.14 +- org.apache.kerby:kerb-core:1.0.1 +- org.apache.kerby:kerby-pkix:1.0.1 +- org.apache.kerby:kerby-asn1:1.0.1 +- org.apache.kerby:kerby-util:1.0.1 +- org.xerial.snappy:snappy-java:1.1.8.3 +- org.wildfly.openssl:wildfly-openssl:1.0.7.Final - software.amazon.ion:ion-java:1.0.2 +This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause). +See bundled license files for details. + +- dnsjava:dnsjava:2.1.7 + +This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT) + +- org.checkerframework:checker-qual:2.5.2 +- org.codehaus.mojo:animal-sniffer-annotations:1.17 + This project bundles the following dependencies under the CDDL 1.1 license. See bundled license files for details. @@ -46,4 +68,4 @@ See bundled license files for details. This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). See bundled license files for details. -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) +- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1) diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE index d133339..da05b23 100644 --- a/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE +++ b/flink-filesystems/flink-s3-fs-presto/src/main/resources/META-INF/NOTICE @@ -27,7 +27,7 @@ This project bundles the following dependencies under the Apache Software Licens - com.fasterxml.jackson.core:jackson-core:2.12.1 - com.fasterxml.jackson.core:jackson-databind:2.12.1 - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.1 -- com.fasterxml.woodstox:woodstox-core:5.0.3 +- com.fasterxml.woodstox:woodstox-core:5.3.0 - com.google.guava:guava:26.0-jre - com.google.inject:guice:4.2.2 - com.facebook.airlift:configuration:0.201 @@ -39,16 +39,30 @@ This project bundles the following dependencies under the Apache Software Licens - org.alluxio:alluxio-shaded-client:2.5.0-3 - org.apache.commons:commons-configuration2:2.1.1 - org.apache.commons:commons-lang3:3.3.2 -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-aws:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 +- org.apache.commons:commons-text:1.4 +- org.apache.hadoop:hadoop-annotations:3.3.1 +- org.apache.hadoop:hadoop-aws:3.3.1 +- org.apache.hadoop:hadoop-auth:3.3.1 +- org.apache.hadoop:hadoop-common:3.3.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1 - org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.httpcomponents:httpcore:4.4.14 - org.apache.httpcomponents:httpclient:4.5.13 +- org.apache.httpcomponents:httpcore:4.4.14 +- org.apache.kerby:kerby-asn1:1.0.1 +- org.apache.kerby:kerb-core:1.0.1 +- org.apache.kerby:kerby-pkix:1.0.1 +- org.apache.kerby:kerby-util:1.0.1 +- org.xerial.snappy:snappy-java:1.1.8.3 - org.weakref:jmxutils:1.19 +- org.wildfly.openssl:wildfly-openssl:1.0.7.Final - software.amazon.ion:ion-java:1.0.2 +This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause). +See bundled license files for details. + +- dnsjava:dnsjava:2.1.7 + This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) See bundled license files for details. @@ -67,7 +81,7 @@ See bundled license files for details. This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). See bundled license files for details. -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) +- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1) This project bundles the following dependencies under the Public Domain. See bundled license files for details. diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml index 3ae6d8f..93957e9 100644 --- a/flink-filesystems/pom.xml +++ b/flink-filesystems/pom.xml @@ -35,7 +35,7 @@ under the License. <packaging>pom</packaging> <properties> - <fs.hadoopshaded.version>3.1.0</fs.hadoopshaded.version> + <fs.hadoopshaded.version>3.3.1</fs.hadoopshaded.version> </properties> <modules>
