This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch 4053-2 in repository https://gitbox.apache.org/repos/asf/storm.git
commit 6ed20a4d25a8e35d523aff75524fcb5c0030b3d4 Author: Richard Zowalla <[email protected]> AuthorDate: Mon Apr 15 08:54:45 2024 +0200 STORM-4053 - Include Client API / Runtime for HDFS Followed advice from Hadoop Devs --- external/storm-hdfs-blobstore/pom.xml | 34 +++++++++++++++--------- external/storm-hdfs/pom.xml | 29 +++++++++++++++------ sql/storm-sql-external/storm-sql-hdfs/pom.xml | 37 ++++++++------------------- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/external/storm-hdfs-blobstore/pom.xml b/external/storm-hdfs-blobstore/pom.xml index 1e566433f..1ef162a73 100644 --- a/external/storm-hdfs-blobstore/pom.xml +++ b/external/storm-hdfs-blobstore/pom.xml @@ -37,12 +37,19 @@ <artifactId>storm-client</artifactId> <version>${project.version}</version> <scope>${provided.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-autocreds</artifactId> + <version>${project.version}</version> <exclusions> - <!--log4j-over-slf4j must be excluded for hadoop-minicluster - see: http://stackoverflow.com/q/20469026/3542091 --> <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> </exclusion> </exclusions> </dependency> @@ -66,10 +73,20 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> + <artifactId>hadoop-client-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-params</artifactId> @@ -94,13 +111,6 @@ <configuration> <reuseForks>false</reuseForks> <forkCount>1</forkCount> - <!-- We exclude these dependencies from the test classpath as they have some ambiguity with classes from the - HDFS mini cluster used for testing. However, users need these dependencies, so we declare it in our POM - --> - <classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-api</classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-runtime</classpathDependencyExcludes> - </classpathDependencyExcludes> </configuration> </plugin> <plugin> diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml index 0f11308b1..f9e0f68cc 100644 --- a/external/storm-hdfs/pom.xml +++ b/external/storm-hdfs/pom.xml @@ -65,6 +65,16 @@ <groupId>org.apache.storm</groupId> <artifactId>storm-autocreds</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -96,10 +106,20 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> + <artifactId>hadoop-client-minicluster</artifactId> <version>${hadoop.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-junit-jupiter</artifactId> @@ -114,13 +134,6 @@ <configuration> <reuseForks>false</reuseForks> <forkCount>1</forkCount> - <!-- We exclude these dependencies from the test classpath as they have some ambiguity with classes from the - HDFS mini cluster used for testing. However, users need these dependencies, so we declare it in our POM - --> - <classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-api</classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-runtime</classpathDependencyExcludes> - </classpathDependencyExcludes> </configuration> </plugin> <plugin> diff --git a/sql/storm-sql-external/storm-sql-hdfs/pom.xml b/sql/storm-sql-external/storm-sql-hdfs/pom.xml index 687abf7e8..88e0406f9 100644 --- a/sql/storm-sql-external/storm-sql-hdfs/pom.xml +++ b/sql/storm-sql-external/storm-sql-hdfs/pom.xml @@ -86,26 +86,18 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> + <artifactId>hadoop-client-minicluster</artifactId> <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>ch.qos.reload4j</groupId> - <artifactId>reload4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-reload4j</artifactId> - </exclusion> - </exclusions> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> <scope>test</scope> </dependency> <dependency> @@ -153,13 +145,6 @@ <systemPropertyVariables> <storm.home>${project.basedir}/target/testhome</storm.home> </systemPropertyVariables> - <!-- We exclude these dependencies from the test classpath as they have some ambiguity with classes from the - HDFS mini cluster used for testing. However, users need these dependencies, so we declare it in our POM - --> - <classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-api</classpathDependencyExcludes> - <classpathDependencyExcludes>org.apache.hadoop:hadoop-client-runtime</classpathDependencyExcludes> - </classpathDependencyExcludes> </configuration> </plugin> </plugins>
