Repository: bigtop Updated Branches: refs/heads/master 611a101aa -> c149934cd
BIGTOP-2706 Exclude hadoop dependencies from apex packaging for 3.5.0 Closes #188 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c149934c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c149934c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c149934c Branch: refs/heads/master Commit: c149934cd9779f01eb6632cd5e4432e1ee8f58ff Parents: 611a101 Author: Chinmay Kolhatkar <[email protected]> Authored: Tue Mar 21 18:43:05 2017 +0530 Committer: Roman Shaposhnik <[email protected]> Committed: Sun Mar 26 22:56:52 2017 -0700 ---------------------------------------------------------------------- .../src/common/apex/do-component-build | 55 ++++++++++++++++++-- .../src/common/apex/patch1-for-hadoop27x.diff | 24 +++++++++ bigtop-packages/src/rpm/apex/SPECS/apex.spec | 3 ++ 3 files changed, 77 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/common/apex/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/apex/do-component-build b/bigtop-packages/src/common/apex/do-component-build index b940275..49a8e82 100644 --- a/bigtop-packages/src/common/apex/do-component-build +++ b/bigtop-packages/src/common/apex/do-component-build @@ -26,10 +26,10 @@ command -v mvn > /dev/null 2>&1 || { } # Build apex -mvn clean package -DskipTests "$@" +mvn clean package -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@" # Find all the dependencies of apex -mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/all-deps -DincludeScope=runtime -DexcludeScope=test +mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/all-deps -DincludeScope=runtime -DexcludeScope=test -Dhadoop.version=$HADOOP_VERSION # Copy apex-engine jar from compiled path cp engine/target/apex-engine.jar all-deps/ @@ -50,12 +50,56 @@ cat > pom-hadoop-deps.xml <<'EOF' <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> - <version>2.2.0</version> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.2.0</version> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <artifactId>commons-beanutils-core</artifactId> + <groupId>commons-beanutils</groupId> + </exclusion> + <exclusion> + <artifactId>jackson-jaxrs</artifactId> + <groupId>org.codehaus.jackson</groupId> + </exclusion> + <exclusion> + <artifactId>jackson-xc</artifactId> + <groupId>org.codehaus.jackson</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-server</artifactId> + <groupId>com.sun.jersey</groupId> + </exclusion> + <exclusion> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> </project> @@ -64,6 +108,7 @@ EOF mvn dependency:copy-dependencies -DoutputDirectory=`pwd`/hadoop-deps \ -DincludeScope=runtime -DexcludeScope=test \ -DexcludeArtifactIds=jersey-client \ + -Dhadoop.version=$HADOOP_VERSION \ -f pom-hadoop-deps.xml # Find the diff between platform and hadoop. And that's what is required for running apex in standalone mode. @@ -71,7 +116,7 @@ mkdir deps for i in `diff -r all-deps/ hadoop-deps/ | grep all-deps | awk '{print $4}'`; do cp all-deps/$i deps/; done rm -rf all-deps/ hadoop-deps/ -rm -rf deps/hadoop-common-2.2.0-tests.jar +rm -rf deps/hadoop-common-$HADOOP_VERSION-tests.jar #Download non-ASF licenses for libraries wget -t 3 -q -O jline-LICENSE.txt https://raw.githubusercontent.com/jline/jline2/jline-2.11/LICENSE.txt http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff b/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff new file mode 100644 index 0000000..2cd6be7 --- /dev/null +++ b/bigtop-packages/src/common/apex/patch1-for-hadoop27x.diff @@ -0,0 +1,24 @@ +diff -Naur orig/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java new/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java +--- orig/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java 2016-11-22 02:49:11.000000000 +0530 ++++ new/engine/src/main/java/com/datatorrent/stram/client/StramAppLauncher.java 2017-03-23 23:08:32.692947303 +0530 +@@ -583,7 +583,7 @@ + if (UserGroupInformation.isSecurityEnabled()) { + long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); + dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime); +- long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); ++ long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT)); + dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime); + setTokenRefreshCredentials(dag, conf); + } +diff -Naur orig/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java new/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java +--- orig/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java 2016-11-22 02:48:16.000000000 +0530 ++++ new/engine/src/main/java/com/datatorrent/stram/plan/logical/LogicalPlan.java 2017-03-23 23:07:18.948865455 +0530 +@@ -156,7 +156,7 @@ + */ + public static Attribute<Boolean> FAST_PUBLISHER_SUBSCRIBER = new Attribute<>(false); + public static Attribute<Long> HDFS_TOKEN_LIFE_TIME = new Attribute<>(604800000L); +- public static Attribute<Long> RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); ++ public static Attribute<Long> RM_TOKEN_LIFE_TIME = new Attribute<>(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + public static Attribute<String> PRINCIPAL = new Attribute<String>(null, new StringCodec.String2String()); + public static Attribute<String> KEY_TAB_FILE = new Attribute<>((String)null, new StringCodec.String2String()); + public static Attribute<Double> TOKEN_REFRESH_ANTICIPATORY_FACTOR = new Attribute<>(0.7); http://git-wip-us.apache.org/repos/asf/bigtop/blob/c149934c/bigtop-packages/src/rpm/apex/SPECS/apex.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/apex/SPECS/apex.spec b/bigtop-packages/src/rpm/apex/SPECS/apex.spec index 210f2e9..3b290e8 100644 --- a/bigtop-packages/src/rpm/apex/SPECS/apex.spec +++ b/bigtop-packages/src/rpm/apex/SPECS/apex.spec @@ -44,6 +44,7 @@ Source1: do-component-build Source2: install_apex.sh Source3: apex.1 Source4: bigtop.bom +#BIGTOP_PATCH_FILES Requires: hadoop-client, bigtop-utils >= 0.7 %description @@ -59,6 +60,8 @@ Apache Apex includes following key features: %prep %setup -n apache-%{name}-core-%{apex_base_version} +#BIGTOP_PATCH_COMMANDS + %build env APEX_VERSION=%{apex_base_version} bash %{SOURCE1}
