SLIDER-598. HBase application package for Windows need to be tested and validated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96c9006d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96c9006d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96c9006d Branch: refs/heads/feature/SLIDER-531-registry-enhancements Commit: 96c9006d40c397c5f0244dfbed5b4c5c9eaece7c Parents: b019271 Author: Sumit Mohanty <[email protected]> Authored: Sat Nov 1 17:31:44 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Sat Nov 1 17:31:44 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase-win/appConfig-default.json | 3 + app-packages/hbase-win/metainfo.xml | 43 ++++++++++++++ .../hbase-win/package/scripts/hbase_rest.py | 62 ++++++++++++++++++++ .../hbase-win/package/scripts/hbase_service.py | 9 +++ .../hbase-win/package/scripts/hbase_thrift.py | 62 ++++++++++++++++++++ .../hbase-win/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++ .../hbase-win/package/scripts/params.py | 31 ++++------ app-packages/hbase-win/resources-default.json | 16 +++++ 8 files changed, 268 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json index 069e2b7..80d20af 100644 --- a/app-packages/hbase-win/appConfig-default.json +++ b/app-packages/hbase-win/appConfig-default.json @@ -15,6 +15,9 @@ "site.global.monitor_protocol": "http", "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;", "site.global.java_library_path": "c:\\hdp\\hadoop\\bin", + "site.global.hbase_rest_port": "17000", + "site.global.hbase_thrift_port": "9090", + "site.global.hbase_thrift2_port": "9091", "site.hbase-env.hbase_master_heapsize": "1024m", "site.hbase-env.hbase_regionserver_heapsize": "1024m", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml index f0c5d4f..da6121d 100644 --- a/app-packages/hbase-win/metainfo.xml +++ b/app-packages/hbase-win/metainfo.xml @@ -41,6 +41,18 @@ <name>org.apache.slider.monitor</name> <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> </export> + <export> + <name>org.apache.slider.hbase.rest</name> + <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift2</name> + <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift</name> + <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value> + </export> </exports> </exportGroup> </exportGroups> @@ -80,12 +92,43 @@ <component> <name>HBASE_REGIONSERVER</name> <category>SLAVE</category> + <minInstanceCount>0</minInstanceCount> <commandScript> <script>scripts/hbase_regionserver.py</script> <scriptType>PYTHON</scriptType> </commandScript> </component> + <component> + <name>HBASE_REST</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports> + <commandScript> + <script>scripts/hbase_rest.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + + <component> + <name>HBASE_THRIFT</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports> + <commandScript> + <script>scripts/hbase_thrift.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + + <component> + <name>HBASE_THRIFT2</name> + <category>MASTER</category> + <minInstanceCount>0</minInstanceCount> + <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports> + <commandScript> + <script>scripts/hbase_thrift2.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> </components> <osSpecifics> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_rest.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_rest.py b/app-packages/hbase-win/package/scripts/hbase_rest.py new file mode 100644 index 0000000..36b51f9 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_rest.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseRest(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='rest') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'rest', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'rest', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-rest.pid") + check_process_status(pid_file) + +if __name__ == "__main__": + HbaseRest().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py index 4a1f103..9a8dc56 100644 --- a/app-packages/hbase-win/package/scripts/hbase_service.py +++ b/app-packages/hbase-win/package/scripts/hbase_service.py @@ -33,6 +33,15 @@ def hbase_service( if name == "regionserver": heap_size = params.regionserver_heapsize main_class = "org.apache.hadoop.hbase.regionserver.HRegionServer" + if name == "rest": + heap_size = params.restserver_heapsize + main_class = "org.apache.hadoop.hbase.rest.RESTServer" + if name == "thrift": + heap_size = params.thriftserver_heapsize + main_class = "org.apache.hadoop.hbase.thrift.ThriftServer" + if name == "thrift2": + heap_size = params.thrift2server_heapsize + main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer" role_user = format("{hbase_user}-{name}") http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_thrift.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift.py b/app-packages/hbase-win/package/scripts/hbase_thrift.py new file mode 100644 index 0000000..84bfc62 --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_thrift.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseThrift(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='thrift') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'thrift', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'thrift', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid") + check_process_status(pid_file) + +if __name__ == "__main__": + HbaseThrift().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_thrift2.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift2.py b/app-packages/hbase-win/package/scripts/hbase_thrift2.py new file mode 100644 index 0000000..b72196c --- /dev/null +++ b/app-packages/hbase-win/package/scripts/hbase_thrift2.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +from hbase import hbase +from hbase_service import hbase_service + + +class HbaseThrift2(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + hbase(name='thrift2') + + def start(self, env): + import params + env.set_params(params) + self.configure(env) # for security + + hbase_service( 'thrift2', + action = 'start' + ) + + def stop(self, env): + import params + env.set_params(params) + + hbase_service( 'thrift2', + action = 'stop' + ) + + def status(self, env): + import status_params + env.set_params(status_params) + pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid") + check_process_status(pid_file) + +if __name__ == "__main__": + HbaseThrift2().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py index 421b9f5..4f55cd0 100644 --- a/app-packages/hbase-win/package/scripts/params.py +++ b/app-packages/hbase-win/package/scripts/params.py @@ -40,40 +40,31 @@ log_dir = config['configurations']['global']['app_log_dir'] input_conf_files_dir = config['configurations']['global']['app_input_conf_dir'] hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] + +""" +Read various ports, unused as of now +""" rest_port = config['configurations']['global']['hbase_rest_port'] thrift_port = config['configurations']['global']['hbase_thrift_port'] thrift2_port = config['configurations']['global']['hbase_thrift2_port'] +""" +Compute or read various heap sizes +""" master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize'] regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize'] regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) +restserver_heapsize = default("configurations/hbase-env/hbase_restserver_heapsize", "512m") +thriftserver_heapsize = default("configurations/hbase-env/hbase_thriftserver_heapsize", "512m") +thrift2server_heapsize = default("configurations/hbase-env/hbase_thrift2server_heapsize", "512m") + hbase_env_sh_template = config['configurations']['hbase-env']['content'] java_library_path = config['configurations']['global']['java_library_path'] hbase_additional_cp = config['configurations']['global']['hbase_additional_cp'] -master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) -regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', - format("{conf_dir}/hbase_regionserver_jaas.conf")) -master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] -regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] - -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -if security_enabled: - _hostname_lowercase = config['hostname'].lower() - master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST', hostname_lowercase) - regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST', hostname_lowercase) - -kinit_path_local = functions.get_kinit_path( - [default("kinit_path_local", None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -if security_enabled: - kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") -else: - kinit_cmd = "" - # log4j.properties if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): log4j_props = config['configurations']['hbase-log4j']['content'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/resources-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json index 4fedf01..91cd640 100644 --- a/app-packages/hbase-win/resources-default.json +++ b/app-packages/hbase-win/resources-default.json @@ -13,11 +13,27 @@ "yarn.memory": "256" }, "slider-appmaster": { + "yarn.memory": "256" }, "HBASE_REGIONSERVER": { "yarn.role.priority": "2", "yarn.component.instances": "1", "yarn.memory": "256" + }, + "HBASE_REST": { + "yarn.role.priority": "3", + "yarn.component.instances": "1", + "yarn.memory": "256" + }, + "HBASE_THRIFT": { + "yarn.role.priority": "4", + "yarn.component.instances": "0", + "yarn.memory": "256" + }, + "HBASE_THRIFT2": { + "yarn.role.priority": "5", + "yarn.component.instances": "1", + "yarn.memory": "256" } } }
