Updated Branches: refs/heads/trunk 8780e308f -> 8372bbff4
AMBARI-3750. Create a HBase region server and master installation/start/stop/check script (Andrew Onischuk via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/8372bbff Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/8372bbff Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/8372bbff Branch: refs/heads/trunk Commit: 8372bbff4dd2f2421e06d87953b7509bcca692e8 Parents: d74bf3d Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Tue Nov 12 17:25:24 2013 +0200 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Tue Nov 12 17:30:53 2013 +0200 ---------------------------------------------------------------------- .../HBASE/package/files/hbaseSmokeVerify.sh | 32 ++++++++ .../services/HBASE/package/scripts/__init__.py | 0 .../services/HBASE/package/scripts/functions.py | 47 +++++++++++ .../services/HBASE/package/scripts/hbase.py | 71 +++++++++++++++++ .../HBASE/package/scripts/hbase_client.py | 60 ++++++-------- .../HBASE/package/scripts/hbase_master.py | 77 +++++++++--------- .../HBASE/package/scripts/hbase_regionserver.py | 80 ++++++++++--------- .../HBASE/package/scripts/hbase_service.py | 60 ++++++-------- .../services/HBASE/package/scripts/params.py | 63 +++++++++++++++ .../HBASE/package/scripts/service_check.py | 69 ++++++++++++++++ ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 +++++++++++++++ ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 +++++++++++++++ .../HBASE/package/templates/hbase-env.sh.j2 | 82 ++++++++++++++++++++ .../HBASE/package/templates/hbase-smoke.sh.j2 | 26 +++++++ .../package/templates/hbase_client_jaas.conf.j2 | 5 ++ .../templates/hbase_grant_permissions.j2 | 21 +++++ .../package/templates/hbase_master_jaas.conf.j2 | 8 ++ .../templates/hbase_regionserver_jaas.conf.j2 | 8 ++ .../HBASE/package/templates/regionservers.j2 | 2 + 19 files changed, 694 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/files/hbaseSmokeVerify.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/files/hbaseSmokeVerify.sh b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/files/hbaseSmokeVerify.sh new file mode 100644 index 0000000..39fe6e5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/files/hbaseSmokeVerify.sh @@ -0,0 +1,32 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +conf_dir=$1 +data=$2 +echo "scan 'ambarismoketest'" | hbase --config $conf_dir shell > /tmp/hbase_chk_verify +cat /tmp/hbase_chk_verify +echo "Looking for $data" +grep -q $data /tmp/hbase_chk_verify +if [ "$?" -ne 0 ] +then + exit 1 +fi + +grep -q '1 row(s)' /tmp/hbase_chk_verify \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/functions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/functions.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/functions.py new file mode 100644 index 0000000..892f6ef --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/functions.py @@ -0,0 +1,47 @@ +import os +import re +import math +import datetime + +from resource_management.core.shell import checked_call + +def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max): + """ + @param heapsize_str: str (e.g '1000m') + @param xmn_percent: float (e.g 0.2) + @param xmn_max: integer (e.g 512) + """ + heapsize = int(re.search('\d+',heapsize_str).group(0)) + heapsize_unit = re.search('\D+',heapsize_str).group(0) + xmn_val = int(math.floor(heapsize*xmn_percent)) + xmn_val -= xmn_val % 8 + + result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val + return str(result_xmn_val) + heapsize_unit + +def get_unique_id_and_date(): + code, out = checked_call("hostid") + id = out.strip() + + now = datetime.datetime.now() + date = now.strftime("%M%d%y") + + return "id{id}_date{date}".format(id=id, date=date) + +def get_kinit_path(pathes_list): + """ + @param pathes: comma separated list + """ + kinit_path = "" + + for x in pathes_list: + if not x: + continue + + path = os.path.join(x,"kinit") + + if os.path.isfile(path): + kinit_path = path + break + + return kinit_path \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase.py new file mode 100644 index 0000000..e19825a --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase.py @@ -0,0 +1,71 @@ +from resource_management import * +import sys + +def hbase(type=None # 'master' or 'regionserver' or 'client' + ): + import params + + Directory( params.conf_dir, + owner = params.hbase_user, + group = params.user_group, + recursive = True + ) + + XmlConfig( "hbase-site.xml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['hbase-site'], + owner = params.hbase_user, + group = params.user_group + ) + + XmlConfig( "hdfs-site.xml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['hdfs-site'], + owner = params.hbase_user, + group = params.user_group + ) + + if 'hbase-policy' in params.config['configurations']: + XmlConfig( "hbase-policy.xml", + configurations = params.config['configurations']['hbase-policy'], + owner = params.hbase_user, + group = params.user_group + ) + # Manually overriding ownership of file installed by hadoop package + else: + File( format("{conf_dir}/hbase-policy.xml"), + owner = params.hbase_user, + group = params.user_group + ) + + hbase_TemplateConfig( 'hbase-env.sh') + + hbase_TemplateConfig( params.metric_prop_file_name, + tag = 'GANGLIA-MASTER' if type == 'master' else 'GANGLIA-RS' + ) + + hbase_TemplateConfig( 'regionservers') + + if params.security_enabled: + hbase_TemplateConfig( format("hbase_{type}_jaas.conf")) + + if type != "client": + Directory( params.pid_dir, + owner = params.hbase_user, + recursive = True + ) + + Directory ( [params.tmp_dir, params.log_dir], + owner = params.hbase_user, + recursive = True + ) + +def hbase_TemplateConfig(name, + tag=None + ): + import params + + TemplateConfig( format("{conf_dir}/{name}"), + owner = params.hbase_user, + template_tag = tag + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py index 9b7cdec..bfd090a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py @@ -1,38 +1,28 @@ -#!/usr/bin/env python2.6 - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - +import sys from resource_management import * -class Client(Script): - - def install(self): - print "Install!" - - def start(self): - print "Start!" - - def stop(self): - print "Stop!" - - def configure(self): - print "Configure!" - +from hbase import hbase + + +class HbaseClient(Script): + def install(self, env): + Package('hbase') + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(type='client') + +#for tests +def main(): + command_type = 'install' + command_data_file = '/root/workspace/HBase/input.json' + basedir = '/root/workspace/HBase/main' + sys.argv = ["", command_type, command_data_file, basedir] + + HbaseClient().execute() + if __name__ == "__main__": - Client().execute() + HbaseClient().execute() http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_master.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_master.py index d9e2ae0..1a1a4ea 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_master.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_master.py @@ -1,38 +1,45 @@ -#!/usr/bin/env python2.6 - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - +import sys from resource_management import * -class Master(Script): - - def install(self): - print "Install!" - - def start(self): - print "Start!" - - def stop(self): - print "Stop!" - - def configure(self): - print "Configure!" - +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseMaster(Script): + def install(self, env): + Package('hbase') + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(type='master') + + def start(self, env): + import params + env.set_params(params) + + hbase_service( 'master', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'master', + action = 'stop' + ) + +def main(): + command_type = sys.argv[1] if len(sys.argv)>1 else "start" + print "Running "+command_type + command_data_file = '/root/workspace/HBase/input.json' + basedir = '/root/workspace/HBase/main' + sys.argv = ["", command_type, command_data_file, basedir] + + HbaseMaster().execute() + if __name__ == "__main__": - Master().execute() + HbaseMaster().execute() http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_regionserver.py index 5073686..757f376 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_regionserver.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_regionserver.py @@ -1,38 +1,48 @@ -#!/usr/bin/env python2.6 - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - +import sys from resource_management import * -class RegionServer(Script): - - def install(self): - print "Install!" - - def start(self): - print "Start!" - - def stop(self): - print "Stop!" - - def configure(self): - print "Configure!" - +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseRegionServer(Script): + def install(self, env): + Package('hbase') + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(type='regionserver') + + def start(self, env): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'regionserver', + action = 'stop' + ) + + def decommission(self): + print "Decommission not yet implemented!" + +def main(): + command_type = sys.argv[1] if len(sys.argv)>1 else "stop" + print "Running "+command_type + command_data_file = '/root/workspace/HBase/input.json' + basedir = '/root/workspace/HBase/main' + sys.argv = ["", command_type, command_data_file, basedir] + + HbaseRegionServer().execute() + if __name__ == "__main__": - RegionServer().execute() + HbaseRegionServer().execute() http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_service.py index b4d5d5b..d8f359f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_service.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_service.py @@ -1,38 +1,26 @@ -#!/usr/bin/env python2.6 - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - from resource_management import * -class Service(Script): - - def install(self): - print "Install!" - - def start(self): - print "Start!" - - def stop(self): - print "Stop!" - - def configure(self): - print "Configure!" - -if __name__ == "__main__": - Service().execute() +def hbase_service( + name, + action = 'start'): # 'start' or 'stop' + + import params + + role = name + cmd = format("{daemon_script} --config {conf_dir}") + pid_file = format("{pid_dir}/hbase-hbase-{role}.pid") + + daemon_cmd = None + no_op_test = None + + if action == 'start': + daemon_cmd = format("{cmd} start {role}") + no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") + elif action == 'stop': + daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}") + + if daemon_cmd != None: + Execute ( daemon_cmd, + not_if = no_op_test, + user = params.hbase_user + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/params.py new file mode 100644 index 0000000..83ff30f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/params.py @@ -0,0 +1,63 @@ +from resource_management import * +import functions + +# server configurations +config = Script.get_config() + +conf_dir = "/etc/hbase/conf" +daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh" + +hbase_user = config['configurations']['global']['hbase_user'] +smokeuser = config['configurations']['global']['smokeuser'] +security_enabled = config['configurations']['global']['security_enabled'] +user_group = config['configurations']['global']['user_group'] + +# this is "hadoop-metrics.properties" for 1.x stacks +metric_prop_file_name = "hadoop-metrics2-hbase.properties" + +# not supporting 32 bit jdk. +java64_home = config['configurations']['global']['java64_home'] + +log_dir = config['configurations']['global']['hbase_log_dir'] +master_heapsize = config['configurations']['global']['hbase_master_heapsize'] + +regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize'] +regionserver_xmn_size = functions.calc_xmn_from_xms(regionserver_heapsize, 0.2, 512) + +pid_dir = config['configurations']['global']['hbase_pid_dir'] +tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] + +client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf")) +master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) +regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf")) +ganglia_server_host = default('ganglia_server_host', "") # is not passed when ganglia is not present +rs_hosts = default('hbase_rs_hosts', config['clusterHostInfo']['slave_hosts']) #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves + +smoke_test_user = config['configurations']['global']['smokeuser'] +smokeuser_permissions = default('smokeuser_permissions', "RWXCA") +service_check_data = functions.get_unique_id_and_date() + +if security_enabled: + + _use_hostname_in_principal = default('instance_name', True) + _master_primary_name = config['configurations']['global']['hbase_master_primary_name'] + _hostname = default('/hostname') + _kerberos_domain = config['configurations']['global']['kerberos_domain'] + _master_principal_name = config['configurations']['global']['hbase_master_principal_name'] + _regionserver_primary_name = config['configurations']['global']['hbase_regionserver_primary_name'] + + if _use_hostname_in_principal: + master_jaas_princ = format("{_master_primary_name}/{_hostname}@{_kerberos_domain}") + regionserver_jaas_princ = format("{_regionserver_primary_name}/{_hostname}@{_kerberos_domain}") + else: + master_jaas_princ = format("{_master_principal_name}@{_kerberos_domain}") + regionserver_jaas_princ = format("{_regionserver_primary_name}@{_kerberos_domain}") + +master_keytab_path = default('configurations/hbase-site/hbase.master.keytab.file') +regionserver_keytab_path = default('/configurations/hbase-site/hbase.regionserver.keytab.file') +smoke_user_keytab = default('smokeuser_keytab') +hbase_user_keytab = default('hbase_user_keytab') +kinit_path_local = functions.get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) + +# fix exeuteHadoop calls for secured cluster +# to string template... \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/service_check.py new file mode 100644 index 0000000..0675d98 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/service_check.py @@ -0,0 +1,69 @@ +from resource_management import * +import functions + + +class HbaseServiceCheck(Script): + def perform(self, env): + import params + env.set_params(params) + + output_file = "/apps/hbase/data/ambarismoketest" + test_cmd = format("fs -test -e {output_file}") + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smoke_test_user};") if params.security_enabled else "" + hbase_servicecheck_file = '/tmp/hbase-smoke.sh' + + File( '/tmp/hbaseSmokeVerify.sh', + content = StaticFile("hbaseSmokeVerify.sh"), + mode = 0755 + ) + + File( hbase_servicecheck_file, + mode = 0755, + content = Template('hbase-smoke.sh.j2') + ) + + if params.security_enabled: + hbase_grant_premissions_file = '/tmp/hbase_grant_permissions.sh' + hbase_kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") + grantprivelegecmd = format("{hbase_kinit_cmd} hbase shell {hbase_grant_premissions_file}") + + File( hbase_grant_premissions_file, + owner = params.hbase_user, + group = params.user_group, + mode = 0644, + content = Template('hbase_grant_permissions.j2') + ) + + Execute( grantprivelegecmd, + user = params.hbase_user, + ) + + servicecheckcmd = format("{kinit_cmd} hbase --config {conf_dir} shell {hbase_servicecheck_file}") + smokeverifycmd = format("{kinit_cmd} /tmp/hbaseSmokeVerify.sh {conf_dir} {service_check_data}") + + Execute( servicecheckcmd, + tries = 3, + try_sleep = 5, + user = params.smoke_test_user, + logoutput = True + ) + + Execute ( smokeverifycmd, + tries = 3, + try_sleep = 5, + user = params.smoke_test_user, + logoutput = True + ) + +def main(): + import sys + command_type = 'perform' + command_data_file = '/root/workspace/HBase/input.json' + basedir = '/root/workspace/HBase/main' + sys.argv = ["", command_type, command_data_file, basedir] + + HbaseServiceCheck().execute() + +if __name__ == "__main__": + HbaseServiceCheck().execute() + http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 new file mode 100644 index 0000000..2583f44 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +hbase.period=10 +hbase.servers={{ganglia_server_host}}:8663 + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +jvm.period=10 +jvm.servers={{ganglia_server_host}}:8663 + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +rpc.period=10 +rpc.servers={{ganglia_server_host}}:8663 + +#Ganglia following hadoop example +hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +hbase.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +hbase.sink.ganglia.servers={{ganglia_server_host}}:8663 http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 new file mode 100644 index 0000000..9f2b616 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +hbase.period=10 +hbase.servers={{ganglia_server_host}}:8660 + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +jvm.period=10 +jvm.servers={{ganglia_server_host}}:8660 + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +rpc.period=10 +rpc.servers={{ganglia_server_host}}:8660 + +#Ganglia following hadoop example +hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +hbase.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +hbase.sink.ganglia.servers={{ganglia_server_host}}:8660 http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-env.sh.j2 new file mode 100644 index 0000000..b8505b5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-env.sh.j2 @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log" +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +{% if security_enabled %} +export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" +{% endif %} http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-smoke.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-smoke.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-smoke.sh.j2 new file mode 100644 index 0000000..61fe62f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase-smoke.sh.j2 @@ -0,0 +1,26 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +disable 'ambarismoketest' +drop 'ambarismoketest' +create 'ambarismoketest','family' +put 'ambarismoketest','row01','family:col01','{{service_check_data}}' +scan 'ambarismoketest' +exit \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_client_jaas.conf.j2 new file mode 100644 index 0000000..696718e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_client_jaas.conf.j2 @@ -0,0 +1,5 @@ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_grant_permissions.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_grant_permissions.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_grant_permissions.j2 new file mode 100644 index 0000000..9102d35 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_grant_permissions.j2 @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +grant '{{smoke_test_user}}', '{{smokeuser_permissions}}' +exit \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_master_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_master_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_master_jaas.conf.j2 new file mode 100644 index 0000000..722cfcc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_master_jaas.conf.j2 @@ -0,0 +1,8 @@ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{master_keytab_path}}" +principal="{{master_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_regionserver_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_regionserver_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_regionserver_jaas.conf.j2 new file mode 100644 index 0000000..cb9b7b0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/hbase_regionserver_jaas.conf.j2 @@ -0,0 +1,8 @@ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{regionserver_keytab_path}}" +principal="{{regionserver_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/8372bbff/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/regionservers.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/regionservers.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/regionservers.j2 new file mode 100644 index 0000000..b22ae5f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/templates/regionservers.j2 @@ -0,0 +1,2 @@ +{% for host in rs_hosts %}{{host}} +{% endfor %} \ No newline at end of file
