Repository: ambari Updated Branches: refs/heads/trunk b1ad9ea73 -> 9ed9f129b
AMBARI-4914 HBase Service Check failed after adding HBase service (Ivan Kozlov via dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ed9f129 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ed9f129 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ed9f129 Branch: refs/heads/trunk Commit: 9ed9f129bdc9858efae0cf19c2ec2911398050e8 Parents: b1ad9ea Author: Dmitry Sen <[email protected]> Authored: Mon Mar 3 18:06:38 2014 +0200 Committer: Dmitry Sen <[email protected]> Committed: Mon Mar 3 18:06:38 2014 +0200 ---------------------------------------------------------------------- .../libraries/functions/__init__.py | 1 + .../libraries/functions/substitute_vars.py | 53 ++++++++++++++ .../resource_management/TestSubstituteVars.py | 75 ++++++++++++++++++++ .../services/HBASE/configuration/hbase-site.xml | 6 ++ .../services/HBASE/configuration/hbase-site.xml | 6 ++ .../services/HBASE/configuration/hbase-site.xml | 6 ++ .../services/HBASE/package/scripts/hbase.py | 7 ++ .../services/HBASE/package/scripts/params.py | 1 + .../stacks/2.0.6/HBASE/test_hbase_client.py | 12 ++++ .../stacks/2.0.6/HBASE/test_hbase_master.py | 12 ++++ .../2.0.6/HBASE/test_hbase_regionserver.py | 12 ++++ .../2.0.6/configs/default.hbasedecom.json | 3 +- .../python/stacks/2.0.6/configs/default.json | 3 +- .../python/stacks/2.0.6/configs/secured.json | 3 +- 14 files changed, 197 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py index dcde79e..32e886b 100644 --- a/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py +++ b/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py @@ -26,3 +26,4 @@ from resource_management.libraries.functions.get_kinit_path import * from resource_management.libraries.functions.get_unique_id_and_date import * from resource_management.libraries.functions.check_process_status import * from resource_management.libraries.functions.is_empty import * +from resource_management.libraries.functions.substitute_vars import * http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py b/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py new file mode 100644 index 0000000..2036208 --- /dev/null +++ b/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import re + +_MAX_SUBST = 20 + +def substitute_vars(raw, config): + """ + @param raw: str (e.g '${hbase.tmp.dir}/local') + @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'}) + """ + result = raw + + pattern = re.compile("\$\{[^\}\$\x0020]+\}") + + for depth in range(0, _MAX_SUBST - 1): + match = pattern.search(result) + + if match: + start = match.start() + end = match.end() + + name = result[start + 2 : end - 1] + + try: + value = config[name] + except KeyError: + return result + + result = result[:start] + value + result[end:] + else: + break + + return result http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py b/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py new file mode 100644 index 0000000..b3623cd --- /dev/null +++ b/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' +from unittest import TestCase, main +from resource_management.libraries.functions.substitute_vars import substitute_vars + +import StringIO, sys + +class TestSubstituteVars(TestCase): + def setUp(self): + # disable stdout + out = StringIO.StringIO() + sys.stdout = out + + def test_substitute_vars(self): + raw_config = { + 'val.intvar' : '42', + 'pass.intvar' : '${val.intvar}', + 'fail.unknown' : 'a${unknown}b', + 'fail.empty' : '${}', + 'fail.space' : '${ my.int}', + 'val.0' : 'will_fail', + 'fail.digit' : '${val.0}', + 'val.file' : 'hello', + 'val.suffix' : '.txt', + 'pass.seq.depth' : '${val.file}${val.suffix}${pass.intvar}', + 'fail.seq.depth.a' : '${val.file}${unknown}${pass.intvar}', + 'fail.seq.depth.b' : '${val.file}${fail.seq.depth.a}${pass.intvar}', + 'val.name' : 'val.intvar', + 'pass.name.as.param' : '${${val.name}}', + 'fail.inf.loop' : '${fail.inf.loop}' + } + expected_config = { + 'val.intvar' : '42', + 'pass.intvar' : '42', + 'fail.unknown' : 'a${unknown}b', + 'fail.empty' : '${}', + 'fail.space' : '${ my.int}', + 'val.0' : 'will_fail', + 'fail.digit' : '${val.0}', + 'val.file' : 'hello', + 'val.suffix' : '.txt', + 'pass.seq.depth' : 'hello.txt42', + 'fail.seq.depth.a' : 'hello${unknown}${pass.intvar}', + 'fail.seq.depth.b' : 'hellohello${unknown}${pass.intvar}${pass.intvar}', + 'val.name' : 'val.intvar', + 'pass.name.as.param' : '42', + 'fail.inf.loop' : '${fail.inf.loop}' + } + + for key in raw_config.keys(): + actual_value = substitute_vars(raw_config[key], raw_config) + expected_value = expected_config[key] + + self.assertEqual(actual_value, expected_value) + + def tearDown(self): + # enable stdout + sys.stdout = sys.__stdout__ http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml index 4244bbc..c1f1b29 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml @@ -52,6 +52,12 @@ </description> </property> <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + </property> + <property> <name>hbase.master.info.bindAddress</name> <value></value> <description>The bind address for the HBase Master web UI http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml index 303a74c..104ed0d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml @@ -52,6 +52,12 @@ </description> </property> <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + </property> + <property> <name>hbase.master.info.bindAddress</name> <value></value> <description>The bind address for the HBase Master web UI http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml index d1e933d..cf9416e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml @@ -52,6 +52,12 @@ </description> </property> <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + </property> + <property> <name>hbase.master.info.bindAddress</name> <value></value> <description>The bind address for the HBase Master web UI http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py index d890339..0646b57 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py @@ -48,6 +48,13 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' recursive = True ) + Directory (os.path.join(params.local_dir, "jars"), + owner = params.hbase_user, + group = params.user_group, + mode=0775, + recursive = True + ) + XmlConfig( "hbase-site.xml", conf_dir = params.conf_dir, configurations = params.config['configurations']['hbase-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py index 3b1316c..1d95320 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py @@ -53,6 +53,7 @@ regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512) pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site']) client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf")) master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py index b497745..50dda17 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py @@ -39,6 +39,12 @@ class TestHBaseClient(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', @@ -96,6 +102,12 @@ class TestHBaseClient(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index f413857..1084ed3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -200,6 +200,12 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', @@ -281,6 +287,12 @@ class TestHBaseMaster(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py index d0eaa90..4ced781 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py @@ -133,6 +133,12 @@ class TestHbaseRegionServer(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', @@ -214,6 +220,12 @@ class TestHbaseRegionServer(RMFTestCase): owner = 'hbase', recursive = True, ) + self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars', + owner = 'hbase', + group = 'hadoop', + mode=0775, + recursive = True, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'hbase', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json index 16d223e..f99821b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json @@ -372,6 +372,7 @@ "hbase.regionserver.global.memstore.upperLimit": "0.4", "zookeeper.session.timeout": "30000", "hbase.tmp.dir": "/hadoop/hbase", + "hbase.local.dir": "${hbase.tmp.dir}/local", "hbase.hregion.max.filesize": "10737418240", "hfile.block.cache.size": "0.40", "hbase.security.authentication": "simple", @@ -609,4 +610,4 @@ "c6402.ambari.apache.org" ] } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index fea2268..db34e87 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -372,6 +372,7 @@ "hbase.regionserver.global.memstore.upperLimit": "0.4", "zookeeper.session.timeout": "30000", "hbase.tmp.dir": "/hadoop/hbase", + "hbase.local.dir": "${hbase.tmp.dir}/local", "hbase.hregion.max.filesize": "10737418240", "hfile.block.cache.size": "0.40", "hbase.security.authentication": "simple", @@ -657,4 +658,4 @@ "c6402.ambari.apache.org" ] } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index a8c9315..8bebaf7 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -487,6 +487,7 @@ "hbase.regionserver.global.memstore.upperLimit": "0.4", "zookeeper.session.timeout": "30000", "hbase.tmp.dir": "/hadoop/hbase", + "hbase.local.dir": "${hbase.tmp.dir}/local", "hfile.block.cache.size": "0.40", "hbase.regionserver.kerberos.principal": "hbase/[email protected]", "hbase.security.authentication": "kerberos", @@ -784,4 +785,4 @@ "c6402.ambari.apache.org" ] } -} \ No newline at end of file +}
