Repository: apex-core Updated Branches: refs/heads/master 5adcd0367 -> 59bdc81f8 (forced update)
APEXCORE-536 #resolve Upgrade Hadoop dependency version from 2.2.0 to 2.6.0 Project: http://git-wip-us.apache.org/repos/asf/apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-core/commit/59bdc81f Tree: http://git-wip-us.apache.org/repos/asf/apex-core/tree/59bdc81f Diff: http://git-wip-us.apache.org/repos/asf/apex-core/diff/59bdc81f Branch: refs/heads/master Commit: 59bdc81f895460d50933026efb8043a76419795d Parents: edad2a8 Author: David Yan <[email protected]> Authored: Mon Oct 3 18:03:59 2016 -0700 Committer: Thomas Weise <[email protected]> Committed: Mon Oct 10 17:29:19 2016 -0700 ---------------------------------------------------------------------- engine/pom.xml | 29 +++++++++++++++----- .../java/com/datatorrent/stram/StramClient.java | 16 ++++------- .../stram/client/StramClientUtils.java | 12 ++++---- pom.xml | 2 +- 4 files changed, 35 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-core/blob/59bdc81f/engine/pom.xml ---------------------------------------------------------------------- diff --git a/engine/pom.xml b/engine/pom.xml index 59c096e..4e005fb 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -181,8 +181,8 @@ <version>0.5</version> <exclusions> <exclusion> - <artifactId>commons-beanutils-core</artifactId> <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils-core</artifactId> </exclusion> </exclusions> </dependency> @@ -217,12 +217,12 @@ <artifactId>jackson-core-asl</artifactId> </exclusion> <exclusion> - <artifactId>jackson-jaxrs</artifactId> <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> </exclusion> <exclusion> - <artifactId>jackson-xc</artifactId> <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> </exclusion> </exclusions> </dependency> @@ -234,8 +234,8 @@ <scope>test</scope> </dependency> <dependency> - <artifactId>jackson-mapper-asl</artifactId> <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> <version>${jackson.version}</version> </dependency> <dependency> @@ -268,12 +268,12 @@ <artifactId>jackson-mapper-asl</artifactId> </exclusion> <exclusion> - <artifactId>commons-beanutils</artifactId> <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils</artifactId> </exclusion> <exclusion> - <artifactId>commons-beanutils-core</artifactId> <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils-core</artifactId> </exclusion> </exclusions> </dependency> @@ -301,8 +301,8 @@ <type>jar</type> <exclusions> <exclusion> - <artifactId>commons-collections</artifactId> <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> </exclusion> </exclusions> </dependency> @@ -388,6 +388,21 @@ <version>1.6.5</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + <version>1.9</version> + <scope>test</scope> + </dependency> + <dependency> + <!-- This is to prevent javax.servlet artifact to be pulled in + as a transitive dependency during the unit tests since this + conflicts with org.eclipse.jetty:jetty-servlet. --> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.5</version> + <scope>provided</scope> + </dependency> </dependencies> <profiles> <profile> http://git-wip-us.apache.org/repos/asf/apex-core/blob/59bdc81f/engine/src/main/java/com/datatorrent/stram/StramClient.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java index 2009b49..89bca14 100644 --- a/engine/src/main/java/com/datatorrent/stram/StramClient.java +++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java @@ -124,7 +124,7 @@ public class StramClient // platform dependencies that are not part of Hadoop and need to be deployed, // entry below will cause containing jar file from client to be copied to cluster - private static final Class<?>[] DATATORRENT_CLASSES = new Class<?>[]{ + private static final Class<?>[] APEX_CLASSES = new Class<?>[]{ com.datatorrent.netlet.util.Slice.class, com.datatorrent.netlet.EventLoop.class, com.datatorrent.bufferserver.server.Server.class, @@ -145,18 +145,14 @@ public class StramClient org.apache.http.message.BasicHeaderValueParser.class, com.esotericsoftware.minlog.Log.class, org.apache.xbean.asm5.tree.ClassNode.class, - // The jersey client inclusion is only for Hadoop-2.2 and should be removed when we upgrade our Hadoop - // dependency version since Hadoop-2.3 onwards has jersey client bundled - com.sun.jersey.api.client.ClientHandler.class, - com.sun.jersey.client.apache4.ApacheHttpClient4Handler.class, org.jctools.queues.SpscArrayQueue.class }; - private static final Class<?>[] DATATORRENT_SECURITY_SPECIFIC_CLASSES = new Class<?>[]{ + private static final Class<?>[] APEX_SECURITY_SPECIFIC_CLASSES = new Class<?>[]{ }; - private static final Class<?>[] DATATORRENT_SECURITY_CLASSES = - (Class<?>[])ArrayUtils.addAll(DATATORRENT_CLASSES, DATATORRENT_SECURITY_SPECIFIC_CLASSES); + private static final Class<?>[] APEX_SECURITY_CLASSES = + (Class<?>[])ArrayUtils.addAll(APEX_CLASSES, APEX_SECURITY_SPECIFIC_CLASSES); public StramClient(Configuration conf, LogicalPlan dag) throws Exception { @@ -326,9 +322,9 @@ public class StramClient if (applicationType.equals(YARN_APPLICATION_TYPE)) { //TODO restrict the security check to only check if security is enabled for webservices. if (UserGroupInformation.isSecurityEnabled()) { - defaultClasses = DATATORRENT_SECURITY_CLASSES; + defaultClasses = APEX_SECURITY_CLASSES; } else { - defaultClasses = DATATORRENT_CLASSES; + defaultClasses = APEX_CLASSES; } } else { throw new IllegalStateException(applicationType + " is not a valid application type."); http://git-wip-us.apache.org/repos/asf/apex-core/blob/59bdc81f/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java index 7a3b8d4..461f057 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java +++ b/engine/src/main/java/com/datatorrent/stram/client/StramClientUtils.java @@ -106,7 +106,7 @@ public class StramClientUtils public static final String SUBDIR_APPS = "apps"; public static final String SUBDIR_PROFILES = "profiles"; public static final String SUBDIR_CONF = "conf"; - public static final int RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE = 10 * 1000; + public static final long RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE = 10 * 1000; public static final String DT_HDFS_TOKEN_MAX_LIFE_TIME = StreamingApplication.DT_PREFIX + "namenode.delegation.token.max-lifetime"; public static final String HDFS_TOKEN_MAX_LIFE_TIME = "dfs.namenode.delegation.token.max-lifetime"; public static final String DT_RM_TOKEN_MAX_LIFE_TIME = StreamingApplication.DT_PREFIX + "resourcemanager.delegation.token.max-lifetime"; @@ -432,15 +432,15 @@ public class StramClientUtils // We are overriding this to be 10 seconds maximum. // - int rmConnectMaxWait = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS); + long rmConnectMaxWait = conf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS); if (rmConnectMaxWait > RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE) { LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.", YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmConnectMaxWait, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE); - conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE); - int rmConnectRetryInterval = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS); - int defaultRetryInterval = Math.max(500, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE / 5); + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE); + long rmConnectRetryInterval = conf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS); + long defaultRetryInterval = Math.max(500, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE / 5); if (rmConnectRetryInterval > defaultRetryInterval) { LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.", YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, rmConnectRetryInterval, defaultRetryInterval); - conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, defaultRetryInterval); + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, defaultRetryInterval); } } LOG.info(" conf object in stramclient {}", conf); http://git-wip-us.apache.org/repos/asf/apex-core/blob/59bdc81f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6107df7..e4c57a3 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.optimize>false</maven.compiler.optimize> <maven.build.timestamp.format>yyyyMMdd</maven.build.timestamp.format> - <hadoop.version>2.2.0</hadoop.version> + <hadoop.version>2.6.0</hadoop.version> <github.global.server>github</github.global.server> <jackson.version>1.9.13</jackson.version> <jersey.version>1.9</jersey.version>
