YARN-7190 Ensure only NM classpath in 2.x gets TSv2 related hbase jars, not the user classpath (Contributed by Varun Saxena via Vrushali C)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b98dba7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b98dba7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b98dba7 Branch: refs/heads/branch-2 Commit: 8b98dba7ee848951f88e0a599a701772fb19547a Parents: 6ba1f34 Author: vrushali <[email protected]> Authored: Mon Oct 16 09:21:31 2017 -0700 Committer: Varun Saxena <[email protected]> Committed: Tue Oct 17 23:36:41 2017 +0530 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 29 +++++++++++++++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 ++++ hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 ++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 1 + .../pom.xml | 25 ++++++++++++- .../hadoop-yarn-server-timelineservice/pom.xml | 1 + hadoop-yarn-project/pom.xml | 38 +++++++++++++++++--- 7 files changed, 100 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a3a05e4..220c127 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -188,9 +188,38 @@ <directory>${project.build.directory}/site</directory> <outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory> </fileSet> + <!-- Copy dependecies from hadoop-yarn-server-timelineservice as well --> + <fileSet> + <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/target/lib</directory> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory> + </fileSet> </fileSets> <moduleSets> <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice</include> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> + <includes> + <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-tests</include> + </includes> + <binaries> + <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/test</outputDirectory> + <includeDependencies>false</includeDependencies> + <unpack>false</unpack> + </binaries> + </moduleSet> + <moduleSet> + <excludes> + <exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice*</exclude> + </excludes> <binaries> <outputDirectory>share/hadoop/${hadoop.component}</outputDirectory> <includeDependencies>false</includeDependencies> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 6ee435e..892d15f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -247,6 +247,8 @@ elif [ "$COMMAND" = "queue" ] ; then YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "resourcemanager" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS" if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then @@ -262,6 +264,8 @@ elif [ "$COMMAND" = "historyserver" ] ; then JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m" fi elif [ "$COMMAND" = "timelinereader" ] ; then + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' elif [ "$COMMAND" = "timelineserver" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/timelineserver-config/log4j.properties @@ -279,6 +283,8 @@ elif [ "$COMMAND" = "sharedcachemanager" ] ; then fi elif [ "$COMMAND" = "nodemanager" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*" + CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS" if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 8415163..708a7c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -219,6 +219,8 @@ goto :eof :resourcemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS% if defined YARN_RESOURCEMANAGER_HEAPSIZE ( @@ -254,11 +256,15 @@ goto :eof :timelinereader set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer goto :eof :nodemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\* set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% if defined YARN_NODEMANAGER_HEAPSIZE ( http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 446fe57..85df2c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -183,6 +183,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-timelineservice</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml index 3358467c..6c568ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml @@ -57,11 +57,13 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-annotations</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> @@ -74,16 +76,19 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-common</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId> + <scope>provided</scope> </dependency> <dependency> @@ -120,6 +125,7 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> @@ -185,6 +191,23 @@ </additionnalDependencies> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> -</project> \ No newline at end of file +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index e0e7659..2597c86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -39,6 +39,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b98dba7/hadoop-yarn-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index a3863dd..1ada81f 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -79,10 +79,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-router</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId> - </dependency> </dependencies> <build> @@ -164,12 +160,44 @@ </execution> </executions> </plugin> + <!-- Remove duplicate dependencies which exist in both yarn/lib folder and yarn/timelineservice/lib folder --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo file="${project.build.directory}/remove-duplicate-timelineservice-jars.sh"> + yarnjardir="${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/yarn" + yarnlibdir="${yarnjardir}/lib/" + cd "${yarnjardir}/timelineservice/lib" + for jar in `ls` + do + if [ -f "${yarnlibdir}""${jar}" ];then + rm -rf ${jar} + echo "Removing duplicate jar $jar from share/hadoop/yarn/timelineservice/lib folder" + fi + done + </echo> + <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true"> + <arg line="./remove-duplicate-timelineservice-jars.sh"/> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </profile> </profiles> - <reporting> <plugins> <plugin> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
