AMBARI-14636 : Add stack advisor validation for AMS distributed properties (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a2a1949 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a2a1949 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a2a1949 Branch: refs/heads/branch-dev-patch-upgrade Commit: 4a2a1949d8ac9759a794d2596b84616864b27145 Parents: ff7e363 Author: Aravindan Vijayan <[email protected]> Authored: Wed Jan 13 16:56:45 2016 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Wed Jan 13 16:56:45 2016 -0800 ---------------------------------------------------------------------- .../0.1.0/configuration/ams-hbase-site.xml | 10 ++++ .../stacks/HDP/2.0.6/services/stack_advisor.py | 52 ++++++++++++++------ .../stacks/2.0.6/common/test_stack_advisor.py | 7 +++ .../stacks/2.2/common/test_stack_advisor.py | 12 +++-- 4 files changed, 62 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2a1949/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml index 6cdffb3..2b69dc5 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml @@ -34,6 +34,10 @@ <type>core-site</type> <name>fs.defaultFS</name> </property> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> </depends-on> </property> <property> @@ -63,6 +67,12 @@ standalone mode and true for distributed mode. If false, startup will run all HBase and ZooKeeper daemons together in the one JVM. </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> </property> <property> <name>hbase.master.wait.on.regionservers.mintostart</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2a1949/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 5c0e30a..ca7d2dd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -485,24 +485,33 @@ class HDP206StackAdvisor(DefaultStackAdvisor): "fs.defaultFS" in services["configurations"]["core-site"]["properties"]: defaultFs = services["configurations"]["core-site"]["properties"]["fs.defaultFS"] + operatingMode = "embedded" + if "ams-site" in services["configurations"]: + if "timeline.metrics.service.operation.mode" in services["configurations"]["ams-site"]["properties"]: + operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"] + + if operatingMode == "distributed": + putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true') + else: + putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false') + rootDir = "file:///var/lib/ambari-metrics-collector/hbase" tmpDir = "/var/lib/ambari-metrics-collector/hbase-tmp" - hbaseClusterDistributed = False zk_port_default = [] if "ams-hbase-site" in services["configurations"]: if "hbase.rootdir" in services["configurations"]["ams-hbase-site"]["properties"]: rootDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.rootdir"] if "hbase.tmp.dir" in services["configurations"]["ams-hbase-site"]["properties"]: tmpDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.tmp.dir"] - if "hbase.cluster.distributed" in services["configurations"]["ams-hbase-site"]["properties"]: - hbaseClusterDistributed = services["configurations"]["ams-hbase-site"]["properties"]["hbase.cluster.distributed"].lower() == 'true' if "hbase.zookeeper.property.clientPort" in services["configurations"]["ams-hbase-site"]["properties"]: zk_port_default = services["configurations"]["ams-hbase-site"]["properties"]["hbase.zookeeper.property.clientPort"] - # Skip recommendation item if default value is present - if hbaseClusterDistributed and not "{{zookeeper_clientPort}}" in zk_port_default: + # Skip recommendation item if default value is present + if operatingMode == "distributed" and not "{{zookeeper_clientPort}}" in zk_port_default: zkPort = self.getZKPort(services) putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", zkPort) + elif operatingMode == "embedded" and not "{{zookeeper_clientPort}}" in zk_port_default: + putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", "61181") mountpoints = ["/"] for collectorHostName in amsCollectorHosts: @@ -519,9 +528,17 @@ class HDP206StackAdvisor(DefaultStackAdvisor): tmpDir = os.path.join(mountpoints[1], tmpDir) else: tmpDir = os.path.join(mountpoints[0], tmpDir) - putAmsHbaseSiteProperty("hbase.rootdir", rootDir) putAmsHbaseSiteProperty("hbase.tmp.dir", tmpDir) + if operatingMode == "distributed": + putAmsHbaseSiteProperty("hbase.rootdir", defaultFs + "/user/ams/hbase") + + if operatingMode == "embedded": + if isLocalRootDir: + putAmsHbaseSiteProperty("hbase.rootdir", rootDir) + else: + putAmsHbaseSiteProperty("hbase.rootdir", "file:///var/lib/ambari-metrics-collector/hbase") + collector_heapsize, hbase_heapsize, total_sinks_count = self.getAmsMemoryRecommendation(services, hosts) putAmsEnvProperty("metrics_collector_heapsize", collector_heapsize) @@ -558,7 +575,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): pass # Distributed mode heap size - if hbaseClusterDistributed: + if operatingMode == "distributed": putHbaseEnvProperty("hbase_master_heapsize", "512") putHbaseEnvProperty("hbase_master_xmn_size", "102") #20% of 512 heap size putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_heapsize) @@ -570,7 +587,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putHbaseEnvProperty("hbase_master_xmn_size", round_to_n(0.15*(hbase_heapsize+hbase_rs_heapsize),64)) # If no local DN in distributed mode - if rootDir.startswith("hdfs://") or (defaultFs.startswith("hdfs://") and rootDir.startswith("/")): + if operatingMode == "distributed": dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE") if set(amsCollectorHosts).intersection(dn_hosts): collector_cohosted_with_dn = "true" @@ -583,7 +600,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package') serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics') sys.path.append(os.path.join(metricsDir, 'scripts')) - mode = 'distributed' if hbaseClusterDistributed else 'embedded' servicesList = [service["StackServices"]["service_name"] for service in services["services"]] from split_points import FindSplitPointsForAMSRegions @@ -604,7 +620,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): ams_hbase_env = configurations["ams-hbase-env"]["properties"] split_point_finder = FindSplitPointsForAMSRegions( - ams_hbase_site, ams_hbase_env, serviceMetricsDir, mode, servicesList) + ams_hbase_site, ams_hbase_env, serviceMetricsDir, operatingMode, servicesList) result = split_point_finder.get_split_points() precision_splits = ' ' @@ -888,17 +904,23 @@ class HDP206StackAdvisor(DefaultStackAdvisor): default_fs = core_site.get("fs.defaultFS") if core_site else "file:///" hbase_rootdir = properties.get("hbase.rootdir") hbase_tmpdir = properties.get("hbase.tmp.dir") + distributed = properties.get("hbase.cluster.distributed") is_local_root_dir = hbase_rootdir.startswith("file://") or (default_fs.startswith("file://") and hbase_rootdir.startswith("/")) + if op_mode == "distributed" and is_local_root_dir: rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to HDFS.") - elif op_mode == "embedded" and hbase_rootdir.startswith('/'): - rootdir_item = self.getWarnItem("In embedded mode schemaless values are not supported for hbase.rootdir") + elif op_mode == "embedded": + if distributed.lower() == "false" and hbase_rootdir.startswith('/') or hbase_rootdir.startswith("hdfs://"): + rootdir_item = self.getWarnItem("In embedded mode hbase.rootdir cannot point to schemaless values or HDFS, " + "Example - file:// for localFS") pass distributed_item = None - distributed = properties.get("hbase.cluster.distributed") - if hbase_rootdir and hbase_rootdir.startswith("hdfs://") and not distributed.lower() == "true": - distributed_item = self.getErrorItem("Distributed property should be set to true if hbase.rootdir points to HDFS.") + if op_mode == "distributed" and not distributed.lower() == "true": + distributed_item = self.getErrorItem("hbase.cluster.distributed property should be set to true for " + "distributed mode") + if op_mode == "embedded" and distributed.lower() == "true": + distributed_item = self.getErrorItem("hbase.cluster.distributed property should be set to false for embedded mode") hbase_zk_client_port = properties.get("hbase.zookeeper.property.clientPort") zkPort = self.getZKPort(services) http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2a1949/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index ec67a82..a584a48 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -1450,6 +1450,13 @@ class TestHDP206StackAdvisor(TestCase): 'level': 'WARN', 'message': 'In distributed mode hbase.rootdir should point to HDFS.', 'type': 'configuration' + }, + { + 'config-name': 'hbase.cluster.distributed', + 'config-type': 'ams-hbase-site', + 'level': 'ERROR', + 'message': 'hbase.cluster.distributed property should be set to true for distributed mode', + 'type': 'configuration' } ] self.assertEquals(res, expected) http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2a1949/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index df627e3..757ef69 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -2042,8 +2042,10 @@ class TestHDP22StackAdvisor(TestCase): "hbase.regionserver.global.memstore.upperLimit": "0.35", "hbase.hregion.memstore.flush.size": "134217728", "hfile.block.cache.size": "0.3", + "hbase.cluster.distributed": "false", "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase", "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp", + "hbase.zookeeper.property.clientPort": "61181", } }, "ams-site": { @@ -2171,6 +2173,8 @@ class TestHDP22StackAdvisor(TestCase): ] services['configurations'] = { + 'core-site': {'properties': {}}, + 'ams-site': {'properties': {}}, 'ams-hbase-site': {'properties': {}}, 'ams-hbase-env': {'properties': {}} } @@ -2211,12 +2215,12 @@ class TestHDP22StackAdvisor(TestCase): "old_value": "512" } ] - services["configurations"]['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase' - services["configurations"]['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true' - expected['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase' + services["configurations"]['ams-site']['properties']['timeline.metrics.service.operation.mode'] = 'distributed' + services["configurations"]["core-site"]["properties"]["fs.defaultFS"] = 'hdfs://host1:8020' + expected['ams-hbase-site']['properties']['hbase.cluster.distributed'] = 'true' + expected['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1:8020/user/ams/hbase' expected['ams-hbase-site']['properties']['hbase.zookeeper.property.clientPort'] = '2181' expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512' - # services["configurations"]['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true' expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true' # Distributed mode, low memory, no splitpoints recommended
