AMBARI-12813 Increase OS kernel parameters for hbase (Juanjo Marron via dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bff61b83 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bff61b83 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bff61b83 Branch: refs/heads/branch-dev-patch-upgrade Commit: bff61b83e962dfc7a8c7c9f8198bd038f16b6ec7 Parents: b3d36fb Author: Dmytro Sen <[email protected]> Authored: Wed Sep 30 17:41:48 2015 +0300 Committer: Dmytro Sen <[email protected]> Committed: Wed Sep 30 17:41:48 2015 +0300 ---------------------------------------------------------------------- .../0.96.0.2.0/configuration/hbase-env.xml | 12 ++++++- .../HBASE/0.96.0.2.0/package/scripts/hbase.py | 20 +++++++++-- .../0.96.0.2.0/package/scripts/params_linux.py | 4 +++ .../0.96.0.2.0/package/scripts/status_params.py | 1 + .../0.96.0.2.0/package/templates/hbase.conf.j2 | 35 ++++++++++++++++++++ .../stacks/2.0.6/HBASE/test_hbase_client.py | 28 ++++++++++++++-- .../stacks/2.0.6/HBASE/test_hbase_master.py | 34 ++++++++++++++++++- .../2.0.6/HBASE/test_hbase_regionserver.py | 33 ++++++++++++++++++ .../2.0.6/HBASE/test_phoenix_queryserver.py | 24 +++++++++++++- 9 files changed, 182 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml index 242e9a7..f1dd64f 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml @@ -86,12 +86,22 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value. <increment-step>256</increment-step> </value-attributes> </property> - <property> + <property> <name>hbase_user</name> <value>hbase</value> <property-type>USER</property-type> <description>HBase User Name.</description> </property> + <property> + <name>hbase_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for HBASE user.</description> + </property> + <property> + <name>hbase_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for HBASE user.</description> + </property> <!-- hbase-env.sh --> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py index 991f4e7..6b5369e 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py @@ -55,7 +55,7 @@ def hbase(name=None): group = params.user_group, recursive = True ) - + parent_dir = os.path.dirname(params.tmp_dir) # In case if we have several placeholders in path while ("${" in parent_dir): @@ -119,8 +119,22 @@ def hbase(name=None): owner = params.hbase_user, content=InlineTemplate(params.hbase_env_sh_template), group = params.user_group, - ) - + ) + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + recursive=True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'hbase.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hbase.conf.j2") + ) + hbase_TemplateConfig( params.metric_prop_file_name, tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS' ) http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index f22b035..635be5e 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -75,6 +75,10 @@ if Script.is_hdp_stack_greater_or_equal("2.2"): hbase_conf_dir = status_params.hbase_conf_dir +limits_conf_dir = status_params.limits_conf_dir + +hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_limit", "32000") +hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000") # no symlink for phoenix-server at this point phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py' http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py index 084ee06..014e8d7 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py @@ -50,5 +50,6 @@ else: tmp_dir = Script.get_tmp_dir() hbase_conf_dir = "/etc/hbase/conf" + limits_conf_dir = "/etc/security/limits.d" if Script.is_hdp_stack_greater_or_equal("2.2"): hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2 new file mode 100644 index 0000000..3580db0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{hbase_user}} - nofile {{hbase_user_nofile_limit}} +{{hbase_user}} - nproc {{hbase_user_nproc_limit}} http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py index 9959874..8d96849 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py @@ -90,6 +90,17 @@ class TestHBaseClient(RMFTestCase): owner = 'hbase', group = 'hadoop' ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-RS', @@ -167,9 +178,20 @@ class TestHBaseClient(RMFTestCase): group = 'hadoop', ) self.assertResourceCalled('File', '/etc/hbase/conf/hbase-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), - owner = 'hbase', - group = 'hadoop', + content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), + owner = 'hbase', + group = 'hadoop', + ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py index 74b4aa6..4ca74a8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py @@ -277,6 +277,17 @@ class TestHBaseMaster(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-MASTER', @@ -393,6 +404,17 @@ class TestHBaseMaster(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-MASTER', @@ -524,7 +546,17 @@ class TestHBaseMaster(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop' ) - + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-master/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-MASTER') http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py index fa134b4..e22584a 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py @@ -177,6 +177,17 @@ class TestHbaseRegionServer(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-RS', @@ -254,6 +265,17 @@ class TestHbaseRegionServer(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', template_tag = 'GANGLIA-RS', @@ -346,6 +368,17 @@ class TestHbaseRegionServer(RMFTestCase): content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop' ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py index 51a9edc..2cb535c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py @@ -271,6 +271,17 @@ class TestPhoenixQueryServer(RMFTestCase): self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', @@ -361,6 +372,17 @@ class TestPhoenixQueryServer(RMFTestCase): self.getConfig()['configurations']['hbase-env']['content']), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/etc/security/limits.d', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf', + content = Template('hbase.conf.j2'), + owner = 'root', + group = 'root', + mode = 0644, + ) self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties', owner = 'hbase', @@ -415,4 +437,4 @@ class TestPhoenixQueryServer(RMFTestCase): ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'phoenix-server', '2.3.0.0-1234'), sudo=True) - self.assertNoMoreResources() \ No newline at end of file + self.assertNoMoreResources()
