This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 64344fb302ecdb7881ba728d36f374a0d7de9746 Author: Chao Sun <[email protected]> AuthorDate: Tue Nov 10 08:00:15 2020 -0800 HADOOP-17324. Don't relocate org.bouncycastle in shaded client jars (#2411) Contributed by Chao Sun. --- hadoop-client-modules/hadoop-client-api/pom.xml | 2 ++ .../resources/ensure-jars-have-correct-contents.sh | 4 +++ .../hadoop-client-integration-tests/pom.xml | 21 ++++++++++++++++ .../apache/hadoop/example/ITUseMiniCluster.java | 10 ++++++++ .../hadoop-client-minicluster/pom.xml | 29 ++++++++++++++-------- .../hadoop-client-runtime/pom.xml | 2 ++ 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml index cd53a7c..f1eb935 100644 --- a/hadoop-client-modules/hadoop-client-api/pom.xml +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -143,6 +143,8 @@ <exclude>org/w3c/dom/**/*</exclude> <exclude>org/xml/sax/*</exclude> <exclude>org/xml/sax/**/*</exclude> + <exclude>org/bouncycastle/*</exclude> + <exclude>org/bouncycastle/**/*</exclude> </excludes> </relocation> <relocation> diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 08f9202..d77424e 100644 --- a/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -43,6 +43,8 @@ allowed_expr+="|^org/apache/hadoop/" allowed_expr+="|^META-INF/" # * whatever under the "webapps" directory; for things shipped by yarn allowed_expr+="|^webapps/" +# * Resources files used by Hadoop YARN mini cluster +allowed_expr+="|^TERMINAL/" # * Hadoop's default configuration files, which have the form # "_module_-default.xml" allowed_expr+="|^[^-]*-default.xml$" @@ -54,6 +56,8 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$" # * Used by JavaSandboxLinuxContainerRuntime as a default, loaded # from root, so can't relocate. :( allowed_expr+="|^java.policy$" +# * Used by javax.annotation +allowed_expr+="|^jndi.properties$" # * allowing native libraries from rocksdb. Leaving native libraries as it is. allowed_expr+="|^librocksdbjni-linux32.so" allowed_expr+="|^librocksdbjni-linux64.so" diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml index 3231961..23fc884 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml +++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml @@ -75,6 +75,27 @@ <artifactId>hadoop-client-minicluster</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>javax.xml.bind</groupId> + <artifactId>jaxb-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>javax.activation</groupId> + <artifactId>activation</artifactId> + <version>1.1.1</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java index 6022fbc..2e30486 100644 --- a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java +++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -43,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; import org.apache.hadoop.hdfs.web.WebHdfsConstants; +import org.apache.hadoop.yarn.server.MiniYARNCluster; /** * Ensure that we can perform operations against the shaded minicluster @@ -54,6 +56,7 @@ public class ITUseMiniCluster { LoggerFactory.getLogger(ITUseMiniCluster.class); private MiniDFSCluster cluster; + private MiniYARNCluster yarnCluster; private static final String TEST_PATH = "/foo/bar/cats/dee"; private static final String FILENAME = "test.file"; @@ -73,6 +76,12 @@ public class ITUseMiniCluster { .numDataNodes(3) .build(); cluster.waitActive(); + + conf.set("yarn.scheduler.capacity.root.queues", "default"); + conf.setInt("yarn.scheduler.capacity.root.default.capacity", 100); + yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); } @After @@ -80,6 +89,7 @@ public class ITUseMiniCluster { if (cluster != null) { cluster.close(); } + IOUtils.cleanupWithLogger(LOG, yarnCluster); } @Test diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 48b6619..56e699f 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -666,10 +666,8 @@ <exclude>junit:junit</exclude> <exclude>com.google.code.findbugs:jsr305</exclude> <exclude>log4j:log4j</exclude> - <exclude>org.eclipse.jetty.websocket:*</exclude> - <exclude>javax.websocket:javax.websocket-api</exclude> - <exclude>javax.annotation:javax.annotation-api</exclude> - <exclude>org.eclipse.jetty:jetty-jndi</exclude> + <exclude>org.eclipse.jetty.websocket:websocket-common</exclude> + <exclude>org.eclipse.jetty.websocket:websocket-api</exclude> <!-- We need a filter that matches just those things that are included in the above artiacts --> <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider --> <exclude>org.bouncycastle:*</exclude> @@ -719,13 +717,6 @@ <exclude>testdata/*</exclude> </excludes> </filter> - <!-- Skip terminal html and javascript --> - <filter> - <artifact>org.apache.hadoop:hadoop-yarn-server-nodemanager:*</artifact> - <excludes> - <exclude>TERMINAL/**/*</exclude> - </excludes> - </filter> <!-- Mockito tries to include its own unrelocated copy of hamcrest. :( --> <filter> @@ -773,6 +764,13 @@ </excludes> </filter> <filter> + <!-- skip jetty license info already incorporated into LICENSE/NOTICE --> + <artifact>org.eclipse.jetty.websocket:*</artifact> + <excludes> + <exclude>about.html</exclude> + </excludes> + </filter> + <filter> <artifact>org.apache.hadoop:*</artifact> <excludes> <!-- No shipping log4j configs in a downstream facing library --> @@ -869,6 +867,8 @@ <exclude>org/w3c/dom/**/*</exclude> <exclude>org/xml/sax/*</exclude> <exclude>org/xml/sax/**/*</exclude> + <exclude>org/bouncycastle/*</exclude> + <exclude>org/bouncycastle/**/*</exclude> </excludes> </relocation> <relocation> @@ -966,6 +966,13 @@ </excludes> </relocation> <relocation> + <pattern>javax/annotation/</pattern> + <shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern> + <excludes> + <exclude>**/pom.xml</exclude> + </excludes> + </relocation> + <relocation> <pattern>jersey/</pattern> <shadedPattern>${shaded.dependency.prefix}.jersey.</shadedPattern> <excludes> diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 80bd1ee..a7cdfe6 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -267,6 +267,8 @@ <exclude>org/w3c/dom/**/*</exclude> <exclude>org/xml/sax/*</exclude> <exclude>org/xml/sax/**/*</exclude> + <exclude>org/bouncycastle/*</exclude> + <exclude>org/bouncycastle/**/*</exclude> </excludes> </relocation> <relocation> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
