[43/50] [abbrv] hadoop git commit: YARN-3663. Federation State and Policy Store (DBMS implementation). (Giovanni Matteo Fumarola via curino).

2017-05-26 Thread subru 
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84ddaf39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 80b00ef..db04592 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -19,13 +19,14 @@ package org.apache.hadoop.yarn.server.federation.store.impl;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Calendar;
 import java.util.List;
+import java.util.TimeZone;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
-import 
org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreErrorCode;
 import 
org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreException;
 import 
org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
 import 
org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
@@ -87,13 +88,26 @@ public abstract class FederationStateStoreBaseTest {
   @Test
   public void testRegisterSubCluster() throws Exception {
 SubClusterId subClusterId = SubClusterId.newInstance("SC");
+
 SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
 
+long previousTimeStamp =
+Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTimeInMillis();
+
 SubClusterRegisterResponse result = stateStore.registerSubCluster(
 SubClusterRegisterRequest.newInstance(subClusterInfo));
 
+long currentTimeStamp =
+Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTimeInMillis();
+
 Assert.assertNotNull(result);
 Assert.assertEquals(subClusterInfo, querySubClusterInfo(subClusterId));
+
+// The saved heartbeat is between the old one and the current timestamp
+Assert.assertTrue(querySubClusterInfo(subClusterId)
+.getLastHeartBeat() <= currentTimeStamp);
+Assert.assertTrue(querySubClusterInfo(subClusterId)
+.getLastHeartBeat() >= previousTimeStamp);
   }
 
   @Test
@@ -120,9 +134,7 @@ public abstract class FederationStateStoreBaseTest {
   stateStore.deregisterSubCluster(deregisterRequest);
   Assert.fail();
 } catch (FederationStateStoreException e) {
-  Assert.assertEquals(
-  FederationStateStoreErrorCode.MEMBERSHIP_UPDATE_DEREGISTER_FAIL,
-  e.getCode());
+  Assert.assertTrue(e.getMessage().startsWith("SubCluster SC not found"));
 }
   }
 
@@ -149,9 +161,8 @@ public abstract class FederationStateStoreBaseTest {
   stateStore.getSubCluster(request).getSubClusterInfo();
   Assert.fail();
 } catch (FederationStateStoreException e) {
-  Assert.assertEquals(
-  FederationStateStoreErrorCode.MEMBERSHIP_SINGLE_SELECT_FAIL,
-  e.getCode());
+  Assert.assertTrue(
+  e.getMessage().startsWith("SubCluster SC does not exist"));
 }
   }
 
@@ -200,13 +211,24 @@ public abstract class FederationStateStoreBaseTest {
 SubClusterId subClusterId = SubClusterId.newInstance("SC");
 registerSubCluster(createSubClusterInfo(subClusterId));
 
+long previousHeartBeat =
+querySubClusterInfo(subClusterId).getLastHeartBeat();
+
 SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
 .newInstance(subClusterId, SubClusterState.SC_RUNNING, "capability");
 stateStore.subClusterHeartbeat(heartbeatRequest);
 
+long currentTimeStamp =
+Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTimeInMillis();
+
 Assert.assertEquals(SubClusterState.SC_RUNNING,
 querySubClusterInfo(subClusterId).getState());
-Assert.assertNotNull(querySubClusterInfo(subClusterId).getLastHeartBeat());
+
+// The saved heartbeat is between the old one and the current timestamp
+Assert.assertTrue(querySubClusterInfo(subClusterId)
+.getLastHeartBeat() <= currentTimeStamp);
+Assert.assertTrue(querySubClusterInfo(subClusterId)
+.getLastHeartBeat() >= previousHeartBeat);
   }
 
   @Test
@@

[43/50] [abbrv] hadoop git commit: YARN-3663. Federation State and Policy Store (DBMS implementation). (Giovanni Matteo Fumarola via curino).

2017-05-16 Thread curino 
YARN-3663. Federation State and Policy Store (DBMS implementation). (Giovanni 
Matteo Fumarola via curino).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88164e20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88164e20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88164e20

Branch: refs/heads/YARN-2915
Commit: 88164e20560dcbd8e6c0388c69b251dc92beeadb
Parents: 305810d
Author: Carlo Curino 
Authored: Tue Apr 25 15:14:02 2017 -0700
Committer: Carlo Curino 
Committed: Tue May 16 08:52:39 2017 -0700

--
 LICENSE.txt |   1 +
 hadoop-project/pom.xml  |  12 +
 .../hadoop/yarn/conf/YarnConfiguration.java |  23 +
 .../yarn/conf/TestYarnConfigurationFields.java  |  14 +
 .../hadoop-yarn-server-common/pom.xml   |  20 +
 .../FederationStateStoreErrorCode.java  | 105 ---
 .../FederationStateStoreException.java  |  17 +-
 .../store/impl/MemoryFederationStateStore.java  |  81 +-
 .../store/impl/SQLFederationStateStore.java | 937 +++
 .../store/records/SubClusterState.java  |  21 +
 ...cationHomeSubClusterStoreInputValidator.java |  12 +-
 ...ationMembershipStateStoreInputValidator.java |  14 +-
 .../FederationPolicyStoreInputValidator.java|   6 +-
 .../store/utils/FederationStateStoreUtils.java  | 109 ++-
 .../impl/FederationStateStoreBaseTest.java  |  74 +-
 .../store/impl/HSQLDBFederationStateStore.java  | 252 +
 .../impl/TestMemoryFederationStateStore.java|   3 +-
 .../store/impl/TestSQLFederationStateStore.java |  49 +
 .../TestFederationStateStoreInputValidator.java | 146 +--
 .../TestFederationStateStoreFacadeRetry.java|   7 +-
 .../FederationStateStoreStoreProcs.sql  | 511 ++
 .../SQLServer/FederationStateStoreTables.sql| 122 +++
 22 files changed, 2228 insertions(+), 308 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88164e20/LICENSE.txt
--
diff --git a/LICENSE.txt b/LICENSE.txt
index 969708f..1582f6c 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -661,6 +661,7 @@ 
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
 
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
 Apache HBase - Server which contains JQuery minified javascript library 
version 1.8.3
+Microsoft SQLServer - JDBC version 6.1.0.jre7
 

 
 Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, 
https://jquery.org/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88164e20/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index cd92fa5..ac121c3 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -100,6 +100,8 @@
 
 1.0.0
 3.0.3
+2.4.11
+6.1.0.jre7
 
 
 1.8
@@ -1280,6 +1282,16 @@
   ehcache
   ${ehcache.version}
 
+
+  com.zaxxer
+  HikariCP-java7
+  ${hikari.version}
+
+
+  com.microsoft.sqlserver
+  mssql-jdbc
+  ${mssql.version}
+
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/88164e20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4ba8c85..7139394 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2592,6 +2592,29 @@ public class YarnConfiguration extends Configuration {
 
   public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
 
+  private static final String FEDERATION_STATESTORE_SQL_PREFIX =
+  FEDERATION_PREFIX + "state-store.sql.";
+
+  public static final String FEDERATION_STATESTORE_SQL_USERNAME =
+  FEDERATION_STATESTORE_SQL_PREFIX + "username";
+
+  public static final String FEDERATION_STATESTORE_SQL_PASSWORD =
+  FEDERATION_STATESTORE_SQL_PREFIX + "password";
+
+  public static final String FEDERATION_STATESTORE_SQL_URL =
+