[04/37] hadoop git commit: YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.

2018-11-06 Thread brahma 
YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster 
unavailability. Contributed by Bilwa S T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aed836ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aed836ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aed836ef

Branch: refs/heads/HDFS-13891
Commit: aed836efbff775d95899d05ff947f1048df8cf19
Parents: babc946
Author: Giovanni Matteo Fumarola 
Authored: Fri Nov 2 11:27:11 2018 -0700
Committer: Giovanni Matteo Fumarola 
Committed: Fri Nov 2 11:27:11 2018 -0700

--
 .../policies/router/LoadBasedRouterPolicy.java  |  6 +++-
 .../router/TestLoadBasedRouterPolicy.java   | 31 
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 06e445b..fa5eb4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import 
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import 
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -95,7 +96,10 @@ public class LoadBasedRouterPolicy extends 
AbstractRouterPolicy {
 }
   }
 }
-
+if (chosen == null) {
+  throw new FederationPolicyException(
+  "Zero Active Subcluster with weight 1.");
+}
 return chosen.toId();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index dc8f99b..58f1b99 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
+import static org.junit.Assert.fail;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -103,4 +105,33 @@ public class TestLoadBasedRouterPolicy extends 
BaseRouterPoliciesTest {
 Assert.assertEquals("sc05", chosen.getId());
   }
 
+  @Test
+  public void testIfNoSubclustersWithWeightOne() {
+setPolicy(new LoadBasedRouterPolicy());
+setPolicyInfo(new WeightedPolicyInfo());
+Map routerWeights = new HashMap<>();
+Map amrmWeights = new HashMap<>();
+// update subcluster with weight 0
+SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", 0));
+SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+sc.toId(), null, null, null, null, -1, SubClusterState.SC_RUNNING, -1,
+generateClusterMetricsInfo(0));
+

[25/48] hadoop git commit: YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.

2018-11-06 Thread brahma 
YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster 
unavailability. Contributed by Bilwa S T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aed836ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aed836ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aed836ef

Branch: refs/heads/HDFS-13532
Commit: aed836efbff775d95899d05ff947f1048df8cf19
Parents: babc946
Author: Giovanni Matteo Fumarola 
Authored: Fri Nov 2 11:27:11 2018 -0700
Committer: Giovanni Matteo Fumarola 
Committed: Fri Nov 2 11:27:11 2018 -0700

--
 .../policies/router/LoadBasedRouterPolicy.java  |  6 +++-
 .../router/TestLoadBasedRouterPolicy.java   | 31 
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 06e445b..fa5eb4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import 
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import 
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -95,7 +96,10 @@ public class LoadBasedRouterPolicy extends 
AbstractRouterPolicy {
 }
   }
 }
-
+if (chosen == null) {
+  throw new FederationPolicyException(
+  "Zero Active Subcluster with weight 1.");
+}
 return chosen.toId();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index dc8f99b..58f1b99 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
+import static org.junit.Assert.fail;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -103,4 +105,33 @@ public class TestLoadBasedRouterPolicy extends 
BaseRouterPoliciesTest {
 Assert.assertEquals("sc05", chosen.getId());
   }
 
+  @Test
+  public void testIfNoSubclustersWithWeightOne() {
+setPolicy(new LoadBasedRouterPolicy());
+setPolicyInfo(new WeightedPolicyInfo());
+Map routerWeights = new HashMap<>();
+Map amrmWeights = new HashMap<>();
+// update subcluster with weight 0
+SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", 0));
+SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+sc.toId(), null, null, null, null, -1, SubClusterState.SC_RUNNING, -1,
+generateClusterMetricsInfo(0));
+

[36/50] [abbrv] hadoop git commit: YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.

2018-11-05 Thread aengineer 
YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster 
unavailability. Contributed by Bilwa S T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aed836ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aed836ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aed836ef

Branch: refs/heads/HDDS-4
Commit: aed836efbff775d95899d05ff947f1048df8cf19
Parents: babc946
Author: Giovanni Matteo Fumarola 
Authored: Fri Nov 2 11:27:11 2018 -0700
Committer: Giovanni Matteo Fumarola 
Committed: Fri Nov 2 11:27:11 2018 -0700

--
 .../policies/router/LoadBasedRouterPolicy.java  |  6 +++-
 .../router/TestLoadBasedRouterPolicy.java   | 31 
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 06e445b..fa5eb4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import 
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import 
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -95,7 +96,10 @@ public class LoadBasedRouterPolicy extends 
AbstractRouterPolicy {
 }
   }
 }
-
+if (chosen == null) {
+  throw new FederationPolicyException(
+  "Zero Active Subcluster with weight 1.");
+}
 return chosen.toId();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index dc8f99b..58f1b99 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
+import static org.junit.Assert.fail;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -103,4 +105,33 @@ public class TestLoadBasedRouterPolicy extends 
BaseRouterPoliciesTest {
 Assert.assertEquals("sc05", chosen.getId());
   }
 
+  @Test
+  public void testIfNoSubclustersWithWeightOne() {
+setPolicy(new LoadBasedRouterPolicy());
+setPolicyInfo(new WeightedPolicyInfo());
+Map routerWeights = new HashMap<>();
+Map amrmWeights = new HashMap<>();
+// update subcluster with weight 0
+SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", 0));
+SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+sc.toId(), null, null, null, null, -1, SubClusterState.SC_RUNNING, -1,
+generateClusterMetricsInfo(0));
+

hadoop git commit: YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability. Contributed by Bilwa S T.

2018-11-02 Thread gifuma 
Repository: hadoop
Updated Branches:
  refs/heads/trunk babc946d4 -> aed836efb


YARN-8897. LoadBasedRouterPolicy throws NPE in case of sub cluster 
unavailability. Contributed by Bilwa S T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aed836ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aed836ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aed836ef

Branch: refs/heads/trunk
Commit: aed836efbff775d95899d05ff947f1048df8cf19
Parents: babc946
Author: Giovanni Matteo Fumarola 
Authored: Fri Nov 2 11:27:11 2018 -0700
Committer: Giovanni Matteo Fumarola 
Committed: Fri Nov 2 11:27:11 2018 -0700

--
 .../policies/router/LoadBasedRouterPolicy.java  |  6 +++-
 .../router/TestLoadBasedRouterPolicy.java   | 31 
 2 files changed, 36 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 06e445b..fa5eb4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import 
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import 
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import 
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -95,7 +96,10 @@ public class LoadBasedRouterPolicy extends 
AbstractRouterPolicy {
 }
   }
 }
-
+if (chosen == null) {
+  throw new FederationPolicyException(
+  "Zero Active Subcluster with weight 1.");
+}
 return chosen.toId();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aed836ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index dc8f99b..58f1b99 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
+import static org.junit.Assert.fail;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -103,4 +105,33 @@ public class TestLoadBasedRouterPolicy extends 
BaseRouterPoliciesTest {
 Assert.assertEquals("sc05", chosen.getId());
   }
 
+  @Test
+  public void testIfNoSubclustersWithWeightOne() {
+setPolicy(new LoadBasedRouterPolicy());
+setPolicyInfo(new WeightedPolicyInfo());
+Map routerWeights = new HashMap<>();
+Map amrmWeights = new HashMap<>();
+// update subcluster with weight 0
+SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", 0));
+SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+sc.toId(), null, null, null, null, -1,