This is an automated email from the ASF dual-hosted git repository. trohrmann pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit 09dae1ff4380b8584c9fc52b8aac32edc96eaa2e Author: Till Rohrmann <trohrm...@apache.org> AuthorDate: Fri May 24 15:32:12 2019 +0200 [FLINK-12115][fs] Decrease dependency footprint of flink-azure-fs-hadoop --- flink-filesystems/flink-azure-fs-hadoop/pom.xml | 92 ++++++++++++++++++------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/flink-filesystems/flink-azure-fs-hadoop/pom.xml b/flink-filesystems/flink-azure-fs-hadoop/pom.xml index 37567ce..781e851 100644 --- a/flink-filesystems/flink-azure-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-azure-fs-hadoop/pom.xml @@ -34,7 +34,6 @@ under the License. <!-- need to use a release which includes this patch: https://github.com/apache/hadoop/commit/02cadbd24bf69925078d044701741e2e3fcb4b2f --> <properties> - <fs.azure.version>2.7.0</fs.azure.version> <fs.azure.sdk.version>1.16.0</fs.azure.sdk.version> <fs.jackson.core.version>2.9.4</fs.jackson.core.version> </properties> @@ -55,16 +54,21 @@ under the License. </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-azure</artifactId> - <version>${fs.azure.version}</version> + <groupId>org.apache.flink</groupId> + <artifactId>flink-fs-hadoop-shaded</artifactId> + <version>${project.version}</version> </dependency> - <!-- for the Azure HDFS related tests --> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop.version}</version> + <artifactId>hadoop-azure</artifactId> + <version>${fs.hadoopshaded.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </exclusion> + </exclusions> </dependency> <!-- for Azure IT tests to check if HTTP endpoints are enabled / not --> @@ -74,18 +78,6 @@ under the License. <version>${fs.azure.sdk.version}</version> <scope>test</scope> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - <version>${fs.jackson.core.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava.version}</version> - <scope>test</scope> - </dependency> <!-- for the behavior test suite --> <dependency> @@ -119,18 +111,70 @@ under the License. </includes> </artifactSet> <relocations> + <!-- relocate the references to Hadoop to match the shaded Hadoop config --> <relocation> <pattern>org.apache.hadoop</pattern> - <shadedPattern>org.apache.flink.fs.shaded.hadoop.org.apache.hadoop</shadedPattern> + <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop</shadedPattern> </relocation> - <!-- relocate the azure-storage dependencies --> + + <!-- shade dependencies internally used by Hadoop and never exposed downstream --> + <relocation> + <pattern>org.apache.commons</pattern> + <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.commons</shadedPattern> + </relocation> + + <!-- relocate the Azure dependencies --> <relocation> - <pattern>com.microsoft.azure.storage</pattern> - <shadedPattern>org.apache.flink.fs.shaded.com.microsoft.azure.storage</shadedPattern> + <pattern>com.microsoft.azure</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.com.microsoft.azure</shadedPattern> + </relocation> + + <!-- shade dependencies internally used by Azure and never exposed downstream --> + <relocation> + <pattern>org.apache.httpcomponents</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.org.apache.httpcomponents</shadedPattern> + </relocation> + <relocation> + <pattern>commons-logging</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.commons-logging</shadedPattern> + </relocation> + <relocation> + <pattern>commons-codec</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.commons-codec</shadedPattern> + </relocation> + <relocation> + <pattern>com.fasterxml</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.com.fasterxml</shadedPattern> + </relocation> + <relocation> + <pattern>com.google</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.com.google</shadedPattern> + </relocation> + <relocation> + <pattern>org.eclipse</pattern> + <shadedPattern>org.apache.flink.fs.azure.shaded.org.eclipse</shadedPattern> + </relocation> + + <!-- shade Flink's Hadoop FS adapter classes --> + <relocation> + <pattern>org.apache.flink.runtime.fs.hdfs</pattern> + <shadedPattern>org.apache.flink.fs.azure.common.hadoop</shadedPattern> + </relocation> + <!-- shade Flink's Hadoop FS utility classes --> + <relocation> + <pattern>org.apache.flink.runtime.util</pattern> + <shadedPattern>org.apache.flink.fs.azure.common</shadedPattern> </relocation> </relocations> <filters> <filter> + <artifact>org.apache.flink:flink-hadoop-fs</artifact> + <excludes> + <exclude>org/apache/flink/runtime/util/HadoopUtils</exclude> + <exclude>org/apache/flink/runtime/fs/hdfs/HadoopRecoverable*</exclude> + </excludes> + </filter> + <filter> <artifact>*</artifact> <excludes> <exclude>properties.dtd</exclude> @@ -141,8 +185,6 @@ under the License. <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> - <exclude>core-default.xml</exclude> - <exclude>hdfs-default.xml</exclude> </excludes> </filter> </filters>