Repository: ambari Updated Branches: refs/heads/branch-2.4 121dc7fa8 -> 7c809e278
AMBARI-17662. Atlas HA fails to come up with error finding ids (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7c809e27 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7c809e27 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7c809e27 Branch: refs/heads/branch-2.4 Commit: 7c809e278e7820e5ff90362209d98bd7d797e086 Parents: 121dc7f Author: Alejandro Fernandez <[email protected]> Authored: Fri Jul 8 17:45:58 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Tue Jul 12 13:39:32 2016 -0700 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 57 ------------ .../ATLAS/0.1.0.2.3/package/scripts/params.py | 91 +++++++++++-------- .../0.1.0.2.3/package/scripts/status_params.py | 13 +-- .../configuration/application-properties.xml | 28 ------ .../BlueprintConfigurationProcessorTest.java | 94 -------------------- .../stacks/2.5/common/test_stack_advisor.py | 4 +- 6 files changed, 63 insertions(+), 224 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index c3af4eb..cf12800 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2129,60 +2129,6 @@ public class BlueprintConfigurationProcessor { } /** - * Custom PropertyUpdater that handles the updating of the Atlas HA related properties. - */ - private static class AtlasHAPropertyUpdater extends MultipleHostTopologyUpdater { - - public AtlasHAPropertyUpdater() { - super("ATLAS_SERVER"); - } - - @Override - public String updateForClusterCreate(String propertyName, String origValue, Map<String, - Map<String, String>> properties, ClusterTopology topology) { - - int serverId = 1; - - StringBuilder sb = new StringBuilder(); - - Collection<String> hosts = topology.getHostAssignmentsForComponent("ATLAS_SERVER"); - - switch (propertyName) { - case "atlas.server.address.id1": - - Map<String, String> applicationProperties = properties.get("application-properties"); - - Boolean ssl_enabled = Boolean.parseBoolean(applicationProperties.get("atlas.enableTLS")); - - String port = ssl_enabled ? applicationProperties.get("atlas.server.https.port") : - applicationProperties.get("atlas.server.http.port"); - - for (String host : hosts) { - - if (serverId > 1) { - sb.append("\n").append("atlas.server.address.id").append(serverId).append("="); - } - sb.append(host).append(":").append(port); - ++serverId; - } - break; - case "atlas.server.ids": - - while (serverId <= hosts.size()) { - if (serverId > 1) { - sb.append(","); - } - sb.append("id" + serverId++); - } - break; - default: - return origValue; - } - return sb.toString(); - } - } - - /** * Custom PropertyUpdater that handles the parsing and updating of the * "templeton.hive.properties" configuration property for WebHCat. * This particular configuration property uses a format of @@ -2660,9 +2606,6 @@ public class BlueprintConfigurationProcessor { // ATLAS atlasPropsMap.put("atlas.server.bind.address", new SingleHostTopologyUpdater("ATLAS_SERVER")); - PropertyUpdater atlasHAUpdater = new AtlasHAPropertyUpdater(); - atlasPropsMap.put("atlas.server.ids", atlasHAUpdater); - atlasPropsMap.put("atlas.server.address.id1", atlasHAUpdater); atlasPropsMap.put("atlas.kafka.bootstrap.servers", new MultipleHostTopologyUpdater("KAFKA_BROKER")); atlasPropsMap.put("atlas.kafka.zookeeper.connect", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); atlasPropsMap.put("atlas.graph.index.search.solr.zookeeper-url", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER", ',', false, true, true)); http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 2119268..8445214 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -17,38 +17,70 @@ See the License for the specific language governing permissions and limitations under the License. """ +# Python Imports import os import sys + +# Local Imports +from status_params import * from resource_management import format_stack_version, Script from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default - -import status_params from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.expect import expect + +def configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled): + """ + Return a dictionary of additional configs to merge if Atlas HA is enabled. + :param atlas_hosts: List of hostnames that contain Atlas + :param metadata_port: Port number + :param is_atlas_ha_enabled: None, True, or False + :return: Dictionary with additional configs to merge to application-properties if HA is enabled. + """ + additional_props = {} + if atlas_hosts is None or len(atlas_hosts) == 0 or metadata_port is None: + return additional_props + + # Sort to guarantee each host sees the same values, assuming restarted at the same time. + atlas_hosts = sorted(atlas_hosts) + + # E.g., id1,id2,id3,...,idn + _server_id_list = ["id" + str(i) for i in range(1, len(atlas_hosts) + 1)] + atlas_server_ids = ",".join(_server_id_list) + additional_props["atlas.server.ids"] = atlas_server_ids + + i = 0 + for curr_hostname in atlas_hosts: + id = _server_id_list[i] + prop_name = "atlas.server.address." + id + prop_value = curr_hostname + ":" + metadata_port + additional_props[prop_name] = prop_value + i += 1 + + # This may override the existing property + if i == 1 or (i > 1 and is_atlas_ha_enabled is False): + additional_props["atlas.server.ha.enabled"] = "false" + elif i > 1: + additional_props["atlas.server.ha.enabled"] = "true" + + return additional_props + # server configurations config = Script.get_config() stack_root = Script.get_stack_root() -tmp_dir = Script.get_tmp_dir() - cluster_name = config['clusterName'] java_version = expect("/hostLevelParams/java_version", int) -# security enabled -security_enabled = status_params.security_enabled - if security_enabled: _hostname_lowercase = config['hostname'].lower() _atlas_principal_name = config['configurations']['application-properties']['atlas.authentication.principal'] atlas_jaas_principal = _atlas_principal_name.replace('_HOST',_hostname_lowercase) atlas_keytab_path = config['configurations']['application-properties']['atlas.authentication.keytab'] -stack_name = status_params.stack_name - # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) @@ -65,8 +97,6 @@ metadata_stop_script = format("{metadata_bin}/atlas_stop.py") # metadata local directory structure log_dir = config['configurations']['atlas-env']['metadata_log_dir'] -conf_dir = status_params.conf_dir # "/etc/metadata/conf" -conf_file = status_params.conf_file # service locations hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf' @@ -74,11 +104,8 @@ hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOM # some commands may need to supply the JAAS location when running as atlas atlas_jaas_file = format("{conf_dir}/atlas_jaas.conf") -# user and status -metadata_user = status_params.metadata_user +# user user_group = config['configurations']['cluster-env']['user_group'] -pid_dir = status_params.pid_dir -pid_file = format("{pid_dir}/atlas.pid") # metadata env java64_home = config['hostLevelParams']['java_home'] @@ -104,6 +131,18 @@ metadata_host = config['hostname'] application_properties = dict(config['configurations']['application-properties']) application_properties['atlas.server.bind.address'] = metadata_host +# Atlas HA should populate +# atlas.server.ids = id1,id2,...,idn +# atlas.server.address.id# = host#:port +atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) +# User should not have to modify this property, but still allow overriding it to False if multiple Atlas servers exist +# This can be None, True, or False +is_atlas_ha_enabled = default("/configurations/application-properties/atlas.server.ha.enabled", None) +additional_ha_props = configs_for_ha(atlas_hosts, metadata_port, is_atlas_ha_enabled) +for k,v in additional_ha_props.iteritems(): + application_properties[k] = v + + metadata_env_content = config['configurations']['atlas-env']['content'] metadata_opts = config['configurations']['atlas-env']['metadata_opts'] @@ -124,7 +163,6 @@ smoke_test_password = 'smoke' smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -kinit_path_local = status_params.kinit_path_local security_check_status_file = format('{log_dir}/security_check.status') if security_enabled: @@ -135,26 +173,6 @@ else: # hbase hbase_conf_dir = "/etc/hbase/conf" -# atlas HA -atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', [])) - -id = 1 -server_ids = "" -server_hosts = "" -first_id = True -for host in atlas_hosts: - server_id = "id" + str(id) - server_host = host + ":" + metadata_port - if first_id: - server_ids = server_id - server_hosts = server_host - else: - server_ids += "," + server_id - server_hosts += "\n" + "atlas.server.address." + server_id + "=" + server_host - - id += 1 - first_id = False - atlas_search_backend = default("/configurations/application-properties/atlas.graph.index.search.backend", "") search_backend_solr = atlas_search_backend.startswith('solr') @@ -199,7 +217,6 @@ atlas_server_max_new_size = config['configurations']['atlas-env']['atlas_server_ if has_ranger_admin and stack_supports_atlas_ranger_plugin: # for create_hdfs_directory - hadoop_bin_dir = status_params.hadoop_bin_dir namenode_host = set(default("/clusterHostInfo/namenode_host", [])) has_namenode = not len(namenode_host) == 0 hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py index 0b0d2ae..7ea70c0 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py @@ -18,8 +18,10 @@ limitations under the License. """ import os + from resource_management import Script -from resource_management.libraries.functions import get_kinit_path, format +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select @@ -27,15 +29,16 @@ from resource_management.libraries.functions import stack_select config = Script.get_config() -conf_file = config['configurations']['atlas-env']['metadata_conf_file'] +conf_file = default("/configurations/atlas-env/metadata_conf_file", "atlas-application.properties") conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' -pid_dir = config['configurations']['atlas-env']['metadata_pid_dir'] +pid_dir = default("/configurations/atlas-env/metadata_pid_dir", "/var/run/atlas") pid_file = format("{pid_dir}/atlas.pid") -metadata_user = config['configurations']['atlas-env']['metadata_user'] + +metadata_user = default("/configurations/atlas-env/metadata_user", None) # Security related/required params hostname = config['hostname'] -security_enabled = config['configurations']['cluster-env']['security_enabled'] +security_enabled = default("/configurations/cluster-env/security_enabled", None) kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) tmp_dir = Script.get_tmp_dir() http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml index 0fe0827..aa9940e 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml @@ -21,34 +21,6 @@ --> <configuration supports_final="false"> - <!-- HA properties --> - <property> - <name>atlas.server.ha.enabled</name> - <value>false</value> - <description>Atlas high availability feature toggle.</description> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>atlas.server.ids</name> - <value>{{server_ids}}</value> - <description>List of Atlas server ids for HA feature.</description> - <value-attributes> - <overridable>false</overridable> - <visible>false</visible> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - <property> - <name>atlas.server.address.id1</name> - <value>{{server_hosts}}</value> - <description>Mapping of Atlas server ids to hosts.</description> - <value-attributes> - <overridable>false</overridable> - <visible>false</visible> - </value-attributes> - <on-ambari-upgrade add="false"/> - </property> - <!-- Misc properties --> <property> <name>atlas.audit.hbase.zookeeper.quorum</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index b595868..692fac6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -4857,100 +4857,6 @@ public class BlueprintConfigurationProcessorTest { } @Test - public void testAtlasHA() throws Exception { - final String expectedHostGroupName = "host_group_1"; - final String host1 = "c6401.ambari.apache.org"; - final String host2 = "c6402.ambari.apache.org"; - final String host3 = "c6403.ambari.apache.org"; - - Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); - Map<String, String> atlasProperties = new HashMap<String, String>(); - properties.put("application-properties", atlasProperties); - - // setup properties that include host information - atlasProperties.put("atlas.server.ids", ""); - atlasProperties.put("atlas.server.address.id1", ""); - atlasProperties.put("atlas.server.http.port", "21000"); - atlasProperties.put("atlas.server.https.port", "21443"); - atlasProperties.put("atlas.enableTLS", "false"); - - Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); - - Collection<String> hgComponents = new HashSet<String>(); - hgComponents.add("ATLAS_SERVER"); - List<String> hosts = new ArrayList<String>(); - hosts.add(host1); - hosts.add(host2); - hosts.add(host3); - TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); - - Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); - hostGroups.add(group1); - - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); - - // call top-level cluster config update method - updater.doUpdateForClusterCreate(); - - assertEquals("id1,id2,id3", atlasProperties.get("atlas.server.ids")); - - List<String> hostArray = - Arrays.asList(atlasProperties.get("atlas.server.address.id1").split("\natlas.server.address.id.=")); - List<String> expected = - Arrays.asList("c6401.ambari.apache.org:21000","c6402.ambari.apache.org:21000", "c6403.ambari.apache.org:21000"); - - Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); - } - - @Test - public void testAtlasHAEnableTLS() throws Exception { - final String expectedHostGroupName = "host_group_1"; - final String host1 = "c6401.ambari.apache.org"; - final String host2 = "c6402.ambari.apache.org"; - final String host3 = "c6403.ambari.apache.org"; - - Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); - Map<String, String> atlasProperties = new HashMap<String, String>(); - properties.put("application-properties", atlasProperties); - - // setup properties that include host information - atlasProperties.put("atlas.server.ids", ""); - atlasProperties.put("atlas.server.address.id1", ""); - atlasProperties.put("atlas.server.http.port", "21000"); - atlasProperties.put("atlas.server.https.port", "21443"); - atlasProperties.put("atlas.enableTLS", "true"); - - Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); - - Collection<String> hgComponents = new HashSet<String>(); - hgComponents.add("ATLAS_SERVER"); - List<String> hosts = new ArrayList<String>(); - hosts.add(host1); - hosts.add(host2); - hosts.add(host3); - TestHostGroup group1 = new TestHostGroup(expectedHostGroupName, hgComponents, hosts); - - Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>(); - hostGroups.add(group1); - - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); - - // call top-level cluster config update method - updater.doUpdateForClusterCreate(); - - assertEquals("id1,id2,id3", atlasProperties.get("atlas.server.ids")); - - List<String> hostArray = - Arrays.asList(atlasProperties.get("atlas.server.address.id1").split("\natlas.server.address.id.=")); - List<String> expected = - Arrays.asList("c6401.ambari.apache.org:21443","c6402.ambari.apache.org:21443", "c6403.ambari.apache.org:21443"); - - Assert.assertTrue(hostArray.containsAll(expected) && expected.containsAll(hostArray)); - } - - @Test public void testHiveConfigClusterUpdateExportedHostGroupValue() throws Exception { final String expectedHostGroupName = "host_group_1"; final String expectedHostName = "c6401.ambari.apache.org"; http://git-wip-us.apache.org/repos/asf/ambari/blob/7c809e27/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 58cce06..40f3367 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -6646,9 +6646,7 @@ class TestHDP25StackAdvisor(TestCase): "atlas.audit.hbase.zookeeper.quorum": "", "atlas.graph.storage.hostname": "", "atlas.kafka.bootstrap.servers": "", - "atlas.kafka.zookeeper.connect": "", - 'atlas.server.address.id1': "", - 'atlas.server.ids': "" + "atlas.kafka.zookeeper.connect": "" } }, "logsearch-solr-env": {
