Repository: hadoop Updated Branches: refs/heads/branch-3.0 61e16bff6 -> 53033c69f
YARN-7190. Ensure only NM classpath in 2.x gets TSv2 related hbase jars, not the user classpath. Contributed by Varun Saxena. (cherry picked from commit 09d996fdd429a85822a06ab87f0e2322d0d7ca68) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53033c69 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53033c69 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53033c69 Branch: refs/heads/branch-3.0 Commit: 53033c69f3cc05660d3af82995b88b3bc4b11bed Parents: 61e16bf Author: Rohith Sharma K S <[email protected]> Authored: Fri Dec 15 21:50:28 2017 +0530 Committer: Rohith Sharma K S <[email protected]> Committed: Fri Dec 15 22:17:43 2017 +0530 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 27 ++++++++++++++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 ++++ hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 ++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 1 + .../pom.xml | 23 ++++++++++++ .../hadoop-yarn-server-timelineservice/pom.xml | 1 + hadoop-yarn-project/pom.xml | 38 +++++++++++++++++--- 7 files changed, 97 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 289061f..4de6240 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -186,10 +186,37 @@ <include>**/*</include> </includes> </fileSet> + <!-- Copy dependecies from hadoop-yarn-server-timelineservice as well --> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/target/lib</directory> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory> + </fileSet> </fileSets> <moduleSets> <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice</include> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-tests</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/test</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> <excludes> + <exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice*</exclude> <exclude>org.apache.hadoop:hadoop-yarn-ui</exclude> </excludes> <binaries> http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 0f19989..cc5a5e5 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -111,6 +111,8 @@ function yarncmd_case ;; nodemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager' # Backwards compatibility if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then @@ -130,6 +132,8 @@ function yarncmd_case ;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' # Backwards compatibility if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then @@ -155,6 +159,8 @@ function yarncmd_case ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; timelineserver) http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index fed3d90..e1ac112 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -219,6 +219,8 @@ goto :eof :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS% if defined YARN_RESOURCEMANAGER_HEAPSIZE ( @@ -248,6 +250,8 @@ goto :eof :timelinereader set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS% goto :eof @@ -260,6 +264,8 @@ goto :eof :nodemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% if defined YARN_NODEMANAGER_HEAPSIZE ( http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index babaf48..f9b9a1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -191,6 +191,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-timelineservice</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml index e54dd40..6859907 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml @@ -57,11 +57,13 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-annotations</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> @@ -75,16 +77,19 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-common</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId> + <scope>provided</scope> </dependency> <dependency> @@ -121,6 +126,7 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -186,6 +192,23 @@ </additionnalDependencies> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index ff94a42..d74f6de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -38,6 +38,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/53033c69/hadoop-yarn-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 7fc16f4..10909ae 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -76,10 +76,6 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-router</artifactId> </dependency> </dependencies> @@ -163,12 +159,44 @@ </execution> </executions> </plugin> + <!-- Remove duplicate dependencies which exist in both yarn/lib folder and yarn/timelineservice/lib folder --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo file="${project.build.directory}/remove-duplicate-timelineservice-jars.sh"> + yarnjardir="${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/yarn" + yarnlibdir="${yarnjardir}/lib/" + cd "${yarnjardir}/timelineservice/lib" + for jar in `ls` + do + if [ -f "${yarnlibdir}""${jar}" ];then + rm -rf ${jar} + echo "Removing duplicate jar $jar from share/hadoop/yarn/timelineservice/lib folder" + fi + done + </echo> + <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true"> + <arg line="./remove-duplicate-timelineservice-jars.sh"/> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </profile> </profiles> - <reporting> <plugins> <plugin> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
