Repository: ambari Updated Branches: refs/heads/trunk 44b87c60d -> d5453695e
AMBARI-5905 - Ambari API consistency for HDP in HA state Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d5453695 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d5453695 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d5453695 Branch: refs/heads/trunk Commit: d5453695e19ff8b4aa7e1fd01318b7d8193f9699 Parents: 44b87c6 Author: Artem Baranchuk <[email protected]> Authored: Wed May 28 01:39:18 2014 +0300 Committer: Artem Baranchuk <[email protected]> Committed: Wed May 28 15:11:07 2014 +0300 ---------------------------------------------------------------------- .../apache/ambari/msi/ClusterDefinition.java | 40 ++++++++++++++- .../apache/ambari/scom/SQLProviderModule.java | 1 + .../ambari/msi/ClusterDefinitionTest.java | 39 ++++++++++++++ .../resources/clusterproperties_HDP21_HA.txt | 54 ++++++++++++++++++++ .../resources/clusterproperties_HDP2_HA.txt | 52 +++++++++++++++++++ 5 files changed, 185 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d5453695/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java index 00b5dd5..fb356a2 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java +++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java @@ -41,6 +41,8 @@ public class ClusterDefinition { private static final String HEADER_TAG = "#"; private static final String HOSTS_HEADER = "hosts"; + private static final String HA_HEADER = "ha settings"; + private static Boolean HA_ENABLE = Boolean.FALSE; private final Set<String> services = new HashSet<String>(); private final Set<String> hosts = new HashSet<String>(); @@ -114,6 +116,7 @@ public class ClusterDefinition { componentNameMap.put("HIVE_SERVER_HOST", hiveComponents); Integer majorStackVersion = getMajorStackVersion(); + Integer minorStackVersion = getMinorStackVersion(); if(majorStackVersion != null) { if(majorStackVersion == 1) { Set<String> mapReduceComponents = new HashSet<String>(); @@ -125,12 +128,20 @@ public class ClusterDefinition { } if(majorStackVersion == 2) { componentNameMap.put("JOURNALNODE_HOST", Collections.singleton("JOURNALNODE")); + componentNameMap.put("HA_JOURNALNODE_HOSTS", Collections.singleton("JOURNALNODE")); + + Set<String> haNamenodeComponents = new HashSet<String>(); + haNamenodeComponents.add("NAMENODE"); + haNamenodeComponents.add("ZKFC"); + componentNameMap.put(minorStackVersion > 0 ? "NN_HA_STANDBY_NAMENODE_HOST" : "HA_NAMENODE_HOST", haNamenodeComponents); Set<String> mapReduce2Components = new HashSet<String>(); mapReduce2Components.add("HISTORYSERVER"); mapReduce2Components.add("RESOURCEMANAGER"); componentNameMap.put("RESOURCEMANAGER_HOST", mapReduce2Components); + componentNameMap.put("RM_HA_STANDBY_RESOURCEMANAGER_HOST", Collections.singleton("RESOURCEMANAGER")); + slaveComponents.add("NODEMANAGER"); //hiveComponents.add("MYSQL_SERVER"); @@ -179,6 +190,7 @@ public class ClusterDefinition { componentServiceMap.put("NODEMANAGER", "YARN"); componentServiceMap.put("RESOURCEMANAGER", "YARN"); componentServiceMap.put("YARN_CLIENT", "YARN"); + componentServiceMap.put("ZKFC", "HDFS"); //componentServiceMap.put("MYSQL_SERVER", "HIVE"); } } @@ -204,12 +216,27 @@ public class ClusterDefinition { try { readClusterDefinition(); + haEnableSetup(); } catch (IOException e) { String msg = "Caught exception reading cluster definition file."; throw new IllegalStateException(msg, e); } } + private void haEnableSetup() { + if(HA_ENABLE) { + Map<String, Set<String>> serviceHostComponents = hostComponents.get(componentServiceMap.get("ZKFC")); + if (serviceHostComponents != null) { + for(String host : serviceHostComponents.keySet()) { + Set<String> hostHostComponents = serviceHostComponents.get(host); + if(hostHostComponents != null && hostHostComponents.contains("NAMENODE")) { + hostHostComponents.add("ZKFC"); + } + } + } + } + } + // ----- ClusterDefinition ------------------------------------------------- @@ -585,6 +612,7 @@ public class ClusterDefinition { String line; boolean hostsSection = false; + boolean haSection = false; while ((line = br.readLine()) != null) { line = line.trim(); @@ -594,13 +622,23 @@ public class ClusterDefinition { String header = line.substring(HEADER_TAG.length()).toLowerCase(); hostsSection = header.equalsIgnoreCase(HOSTS_HEADER); + haSection = header.equalsIgnoreCase(HA_HEADER); if (!hostsSection && (header.startsWith(HOSTS_HEADER) ) ){ char c = header.charAt(HOSTS_HEADER.length()); hostsSection = c == ' ' || c == '('; } + + if(!haSection && header.startsWith(HA_HEADER)) { + char c = header.charAt(HA_HEADER.length()); + haSection = c == ' ' || c == '('; + } } else { - if (hostsSection) { + if (hostsSection || haSection) { + + if(haSection && line.toUpperCase().contains("HA=YES")) { + HA_ENABLE = Boolean.TRUE; + } int i = line.indexOf('='); if (i > -1) { http://git-wip-us.apache.org/repos/asf/ambari/blob/d5453695/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java index b5cdb09..a62f3f7 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java +++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java @@ -79,6 +79,7 @@ public class SQLProviderModule extends DefaultProviderModule implements HostInfo serviceNames.put("HBASE_MASTER", "master"); serviceNames.put("HBASE_REGIONSERVER", "regionserver"); serviceNames.put("ZOOKEEPER_SERVER", "zkServer"); + serviceNames.put("ZKFC", "zkfc"); serviceNames.put("DATANODE", "datanode"); serviceNames.put("TASKTRACKER", "tasktracker"); serviceNames.put("WEBHCAT_SERVER", "templeton"); http://git-wip-us.apache.org/repos/asf/ambari/blob/d5453695/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java index 31d79de..e8d15f4 100644 --- a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java +++ b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java @@ -100,6 +100,17 @@ public class ClusterDefinitionTest { components = clusterDefinition.getComponents("HIVE"); Assert.assertTrue(components.contains("HIVE_SERVER")); + + clusterDefinition = new ClusterDefinition(new TestStateProvider(), new TestClusterDefinitionProvider("clusterproperties_HDP2_HA.txt", "myCluster", "HDP-2.0.6"), new TestHostInfoProvider()); + components = clusterDefinition.getComponents("HDFS"); + Assert.assertTrue(components.contains("NAMENODE")); + Assert.assertTrue(components.contains("SECONDARY_NAMENODE")); + Assert.assertTrue(components.contains("DATANODE")); + Assert.assertTrue(components.contains("ZKFC")); + + clusterDefinition = new ClusterDefinition(new TestStateProvider(), new TestClusterDefinitionProvider("clusterproperties_HDP21_HA.txt", "myCluster", "HDP-2.1.2"), new TestHostInfoProvider()); + components = clusterDefinition.getComponents("YARN"); + Assert.assertTrue(components.contains("RESOURCEMANAGER")); } @Test @@ -117,6 +128,34 @@ public class ClusterDefinitionTest { hostComponents = clusterDefinition.getHostComponents("HDFS", "slave2.acme.com"); Assert.assertTrue(hostComponents.contains("DATANODE")); + + clusterDefinition = new ClusterDefinition(new TestStateProvider(), new TestClusterDefinitionProvider("clusterproperties_HDP2_HA.txt", "myCluster", "HDP-2.0.6"), new TestHostInfoProvider()); + hostComponents = clusterDefinition.getHostComponents("HDFS", "WINHDP-1"); + Assert.assertTrue(hostComponents.contains("NAMENODE")); + Assert.assertTrue(hostComponents.contains("JOURNALNODE")); + Assert.assertTrue(hostComponents.contains("ZKFC")); + Assert.assertFalse(hostComponents.contains("DATANODE")); + Assert.assertFalse(hostComponents.contains("SECONDARY_NAMENODE")); + + hostComponents = clusterDefinition.getHostComponents("HDFS", "WINHDP-2"); + Assert.assertTrue(hostComponents.contains("NAMENODE")); + Assert.assertTrue(hostComponents.contains("JOURNALNODE")); + Assert.assertTrue(hostComponents.contains("ZKFC")); + Assert.assertTrue(hostComponents.contains("DATANODE")); + Assert.assertTrue(hostComponents.contains("SECONDARY_NAMENODE")); + + clusterDefinition = new ClusterDefinition(new TestStateProvider(), new TestClusterDefinitionProvider("clusterproperties_HDP21_HA.txt", "myCluster", "HDP-2.1.2"), new TestHostInfoProvider()); + hostComponents = clusterDefinition.getHostComponents("YARN", "WINHDP-1"); + Assert.assertTrue(hostComponents.contains("RESOURCEMANAGER")); + + hostComponents = clusterDefinition.getHostComponents("HDFS", "WINHDP-1"); + Assert.assertTrue(hostComponents.contains("NAMENODE")); + + hostComponents = clusterDefinition.getHostComponents("YARN", "WINHDP-2"); + Assert.assertTrue(hostComponents.contains("RESOURCEMANAGER")); + + hostComponents = clusterDefinition.getHostComponents("HDFS", "WINHDP-2"); + Assert.assertTrue(hostComponents.contains("NAMENODE")); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/d5453695/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP21_HA.txt ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP21_HA.txt b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP21_HA.txt new file mode 100644 index 0000000..4386df2 --- /dev/null +++ b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP21_HA.txt @@ -0,0 +1,54 @@ +### Licensed to the Apache Software Foundation (ASF) under one or more +### contributor license agreements. See the NOTICE file distributed with +### this work for additional information regarding copyright ownership. +### The ASF licenses this file to You under the Apache License, Version 2.0 +### (the "License"); you may not use this file except in compliance with +### the License. You may obtain a copy of the License at +### +### http://www.apache.org/licenses/LICENSE-2.0 +### +### Unless required by applicable law or agreed to in writing, software +### distributed under the License is distributed on an "AS IS" BASIS, +### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +### See the License for the specific language governing permissions and +### limitations under the License. + +#Log directory +HDP_LOG_DIR=c:\hadoop\logs + +#Data directory +HDP_DATA_DIR=c:\hdpdata + +#hosts +NAMENODE_HOST=WINHDP-1 +RESOURCEMANAGER_HOST=WINHDP-1 +HIVE_SERVER_HOST=WINHDP-1 +OOZIE_SERVER_HOST=WINHDP-1 +WEBHCAT_HOST=WINHDP-2 +SLAVE_HOSTS=WINHDP-2 +ZOOKEEPER_HOSTS=WINHDP-1,WINHDP-2 +CLIENT_HOSTS=WINHDP-2 +#HA settings +HA=yes +NN_HA_JOURNALNODE_HOSTS=WINHDP-1,WINHDP-2 +NN_HA_CLUSTER_NAME=nn-ha +NN_HA_JOURNALNODE_EDITS_DIR=C:\hdp\journal +NN_HA_STANDBY_NAMENODE_HOST=WINHDP-2 +RM_HA_CLUSTER_NAME=rm-ha +RM_HA_STANDBY_RESOURCEMANAGER_HOST=WINHDP-2 + +#Database host +DB_FLAVOR=MSSQL +DB_HOSTNAME=WINHDP-1 +DB_PORT=1433 + +#Hive properties +HIVE_DB_NAME=hive +HIVE_DB_USERNAME=hive +HIVE_DB_PASSWORD=hive + +#Oozie properties +OOZIE_DB_NAME=oozie +OOZIE_DB_USERNAME=oozie +OOZIE_DB_PASSWORD=oozie + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5453695/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2_HA.txt ---------------------------------------------------------------------- diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2_HA.txt b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2_HA.txt new file mode 100644 index 0000000..99cf53a --- /dev/null +++ b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2_HA.txt @@ -0,0 +1,52 @@ +### Licensed to the Apache Software Foundation (ASF) under one or more +### contributor license agreements. See the NOTICE file distributed with +### this work for additional information regarding copyright ownership. +### The ASF licenses this file to You under the Apache License, Version 2.0 +### (the "License"); you may not use this file except in compliance with +### the License. You may obtain a copy of the License at +### +### http://www.apache.org/licenses/LICENSE-2.0 +### +### Unless required by applicable law or agreed to in writing, software +### distributed under the License is distributed on an "AS IS" BASIS, +### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +### See the License for the specific language governing permissions and +### limitations under the License. + +#Log directory +HDP_LOG_DIR=c:\hadoop\logs + +#Data directory +HDP_DATA_DIR=c:\hdpdata + +#hosts +NAMENODE_HOST=WINHDP-1 +SECONDARY_NAMENODE_HOST=WINHDP-2 +RESOURCEMANAGER_HOST=WINHDP-1 +HIVE_SERVER_HOST=WINHDP-1 +OOZIE_SERVER_HOST=WINHDP-1 +WEBHCAT_HOST=WINHDP-2 +SLAVE_HOSTS=WINHDP-2 +CLIENT_HOSTS=WINHDP-2 +#HA settings +HA=yes +HA_JOURNALNODE_HOSTS=WINHDP-2,WINHDP-1 +HA_CLUSTER_NAME=hdp2-ha +HA_JOURNALNODE_EDITS_DIR=c:\hadoop\journal +HA_NAMENODE_HOST=WINHDP-2 +ZOOKEEPER_HOSTS=WINHDP-1,WINHDP-2 + +#Database host +DB_FLAVOR=MSSQL +DB_HOSTNAME=WINHDP-1 +DB_PORT=1433 + +#Hive properties +HIVE_DB_NAME=hive +HIVE_DB_USERNAME=hive +HIVE_DB_PASSWORD=hive + +#Oozie properties +OOZIE_DB_NAME=oozie +OOZIE_DB_USERNAME=oozie +OOZIE_DB_PASSWORD=oozie
