AMBARI-18739. Perf: Create Rolling and Express Upgrade Packs (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2c362fd0 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2c362fd0 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2c362fd0 Branch: refs/heads/trunk Commit: 2c362fd0fedb63831ea890bfda984b1cde56d216 Parents: 6ba9bd0 Author: Lisnichenko Dmitro <[email protected]> Authored: Fri Jan 20 12:15:31 2017 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Fri Jan 20 12:16:22 2017 +0200 ---------------------------------------------------------------------- .../main/python/ambari_agent/PythonExecutor.py | 19 +- .../libraries/script/dummy.py | 29 +- .../custom_actions/scripts/ru_set_all.py | 2 + .../hooks/before-INSTALL/scripts/conf-select.py | 35 + .../before-INSTALL/scripts/distro-select.py | 145 + .../1.0/hooks/before-INSTALL/scripts/hook.py | 33 +- .../1.0/hooks/before-INSTALL/scripts/params.py | 23 + .../main/resources/stacks/PERF/1.0/metainfo.xml | 6 +- .../stacks/PERF/1.0/repos/repoinfo.xml | 8 +- .../PERF/1.0/services/FAKEHBASE/alerts.json | 35 + .../configuration/hbase-alert-config.xml | 80 + .../FAKEHBASE/configuration/hbase-env.xml | 292 + .../FAKEHBASE/configuration/hbase-log4j.xml | 146 + .../configuration/hbase-logsearch-conf.xml | 111 + .../FAKEHBASE/configuration/hbase-policy.xml | 53 + .../FAKEHBASE/configuration/hbase-site.xml | 555 ++ .../configuration/ranger-hbase-audit.xml | 122 + .../ranger-hbase-policymgr-ssl.xml | 66 + .../configuration/ranger-hbase-security.xml | 68 + .../PERF/1.0/services/FAKEHBASE/kerberos.json | 159 + .../PERF/1.0/services/FAKEHBASE/metainfo.xml | 197 + .../PERF/1.0/services/FAKEHBASE/metrics.json | 9374 ++++++++++++++++++ .../package/alerts/hbase_master_process.py | 59 + .../alerts/hbase_regionserver_process.py | 59 + .../FAKEHBASE/package/scripts/hbase_client.py | 38 + .../FAKEHBASE/package/scripts/hbase_master.py | 45 + .../package/scripts/hbase_regionserver.py | 45 + .../package/scripts/phoenix_queryserver.py | 42 + .../FAKEHBASE/package/scripts/service_check.py | 30 + .../FAKEHBASE/quicklinks/quicklinks.json | 97 + .../1.0/services/FAKEHBASE/themes/theme.json | 411 + .../PERF/1.0/services/FAKEHBASE/widgets.json | 510 + .../PERF/1.0/services/FAKEHDFS/alerts.json | 120 + .../FAKEHDFS/configuration/core-site.xml | 225 + .../FAKEHDFS/configuration/hadoop-env.xml | 419 + .../hadoop-metrics2.properties.xml | 125 + .../FAKEHDFS/configuration/hadoop-policy.xml | 130 + .../configuration/hdfs-alert-config.xml | 80 + .../FAKEHDFS/configuration/hdfs-log4j.xml | 225 + .../configuration/hdfs-logsearch-conf.xml | 248 + .../FAKEHDFS/configuration/hdfs-site.xml | 633 ++ .../configuration/ranger-hdfs-audit.xml | 124 + .../ranger-hdfs-plugin-properties.xml | 88 + .../configuration/ranger-hdfs-policymgr-ssl.xml | 67 + .../configuration/ranger-hdfs-security.xml | 65 + .../FAKEHDFS/configuration/ssl-client.xml | 70 + .../FAKEHDFS/configuration/ssl-server.xml | 80 + .../PERF/1.0/services/FAKEHDFS/kerberos.json | 246 + .../PERF/1.0/services/FAKEHDFS/metainfo.xml | 266 + .../PERF/1.0/services/FAKEHDFS/metrics.json | 7905 +++++++++++++++ .../package/alerts/alert_checkpoint_time.py | 59 + .../alerts/alert_datanode_unmounted_data_dir.py | 59 + .../package/alerts/alert_nfs_gateway_process.py | 59 + .../package/alerts/alert_snamenode_process.py | 59 + .../package/alerts/alert_upgrade_finalized.py | 59 + .../FAKEHDFS/package/scripts/datanode.py | 57 + .../FAKEHDFS/package/scripts/hdfs_client.py | 38 + .../FAKEHDFS/package/scripts/journalnode.py | 58 + .../FAKEHDFS/package/scripts/namenode.py | 79 + .../FAKEHDFS/package/scripts/nfsgateway.py | 42 + .../services/FAKEHDFS/package/scripts/params.py | 33 + .../FAKEHDFS/package/scripts/service_check.py | 30 + .../FAKEHDFS/package/scripts/snamenode.py | 42 + .../FAKEHDFS/package/scripts/zkfc_slave.py | 38 + .../FAKEHDFS/quicklinks/quicklinks.json | 76 + .../1.0/services/FAKEHDFS/themes/theme.json | 179 + .../PERF/1.0/services/FAKEHDFS/widgets.json | 649 ++ .../1.0/services/FAKEYARN/YARN_metrics.json | 3486 +++++++ .../1.0/services/FAKEYARN/YARN_widgets.json | 611 ++ .../PERF/1.0/services/FAKEYARN/alerts.json | 77 + .../configuration-mapred/mapred-env.xml | 50 + .../configuration-mapred/mapred-site.xml | 134 + .../configuration/capacity-scheduler.xml | 69 + .../configuration/ranger-yarn-audit.xml | 121 + .../ranger-yarn-plugin-properties.xml | 82 + .../configuration/ranger-yarn-policymgr-ssl.xml | 66 + .../configuration/ranger-yarn-security.xml | 58 + .../configuration/yarn-alert-config.xml | 80 + .../FAKEYARN/configuration/yarn-env.xml | 201 + .../FAKEYARN/configuration/yarn-log4j.xml | 103 + .../FAKEYARN/configuration/yarn-site.xml | 780 ++ .../PERF/1.0/services/FAKEYARN/kerberos.json | 278 + .../PERF/1.0/services/FAKEYARN/metainfo.xml | 357 + .../package/alerts/alert_history_process.py | 59 + .../package/alerts/alert_nodemanager_health.py | 59 + .../alerts/alert_resourcemanager_process.py | 59 + .../package/alerts/alert_timeline_process.py | 59 + .../scripts/application_timeline_server.py | 42 + .../FAKEYARN/package/scripts/historyserver.py | 42 + .../package/scripts/mapred_service_check.py | 30 + .../package/scripts/mapreduce2_client.py | 38 + .../FAKEYARN/package/scripts/nodemanager.py | 42 + .../FAKEYARN/package/scripts/resourcemanager.py | 48 + .../FAKEYARN/package/scripts/service_check.py | 30 + .../FAKEYARN/package/scripts/yarn_client.py | 38 + .../FAKEYARN/quicklinks-mapred/quicklinks.json | 76 + .../FAKEYARN/quicklinks/quicklinks.json | 76 + .../services/FAKEYARN/themes-mapred/theme.json | 132 + .../1.0/services/FAKEYARN/themes/theme.json | 250 + .../PERF/1.0/services/FAKEZOOKEEPER/alerts.json | 20 + .../configuration/zk-alert-config.xml | 80 + .../1.0/services/FAKEZOOKEEPER/kerberos.json | 39 + .../1.0/services/FAKEZOOKEEPER/metainfo.xml | 69 + .../package/alerts/alert_zk_server_process.py | 59 + .../package/scripts/service_check.py | 30 + .../package/scripts/zookeeper_client.py | 38 + .../package/scripts/zookeeper_server.py | 42 + .../PERF/1.0/services/GRUMPY/kerberos.json | 2 +- .../PERF/1.0/services/GRUMPY/metainfo.xml | 1 + .../PERF/1.0/services/HAPPY/kerberos.json | 2 +- .../stacks/PERF/1.0/services/HAPPY/metainfo.xml | 1 + .../stacks/PERF/1.0/services/HBASE/alerts.json | 35 - .../HBASE/configuration/hbase-alert-config.xml | 80 - .../services/HBASE/configuration/hbase-env.xml | 292 - .../HBASE/configuration/hbase-log4j.xml | 146 - .../configuration/hbase-logsearch-conf.xml | 111 - .../HBASE/configuration/hbase-policy.xml | 53 - .../services/HBASE/configuration/hbase-site.xml | 573 -- .../HBASE/configuration/ranger-hbase-audit.xml | 122 - .../ranger-hbase-policymgr-ssl.xml | 66 - .../configuration/ranger-hbase-security.xml | 68 - .../PERF/1.0/services/HBASE/kerberos.json | 159 - .../stacks/PERF/1.0/services/HBASE/metainfo.xml | 197 - .../stacks/PERF/1.0/services/HBASE/metrics.json | 9374 ------------------ .../package/alerts/hbase_master_process.py | 59 - .../alerts/hbase_regionserver_process.py | 59 - .../HBASE/package/scripts/hbase_client.py | 38 - .../HBASE/package/scripts/hbase_master.py | 45 - .../HBASE/package/scripts/hbase_regionserver.py | 45 - .../package/scripts/phoenix_queryserver.py | 42 - .../HBASE/package/scripts/service_check.py | 30 - .../services/HBASE/quicklinks/quicklinks.json | 97 - .../PERF/1.0/services/HBASE/themes/theme.json | 411 - .../stacks/PERF/1.0/services/HBASE/widgets.json | 510 - .../stacks/PERF/1.0/services/HDFS/alerts.json | 120 - .../services/HDFS/configuration/core-site.xml | 225 - .../services/HDFS/configuration/hadoop-env.xml | 419 - .../hadoop-metrics2.properties.xml | 125 - .../HDFS/configuration/hadoop-policy.xml | 130 - .../HDFS/configuration/hdfs-alert-config.xml | 80 - .../services/HDFS/configuration/hdfs-log4j.xml | 225 - .../HDFS/configuration/hdfs-logsearch-conf.xml | 248 - .../services/HDFS/configuration/hdfs-site.xml | 633 -- .../HDFS/configuration/ranger-hdfs-audit.xml | 124 - .../ranger-hdfs-plugin-properties.xml | 88 - .../configuration/ranger-hdfs-policymgr-ssl.xml | 67 - .../HDFS/configuration/ranger-hdfs-security.xml | 65 - .../services/HDFS/configuration/ssl-client.xml | 70 - .../services/HDFS/configuration/ssl-server.xml | 80 - .../stacks/PERF/1.0/services/HDFS/kerberos.json | 246 - .../stacks/PERF/1.0/services/HDFS/metainfo.xml | 266 - .../stacks/PERF/1.0/services/HDFS/metrics.json | 7905 --------------- .../package/alerts/alert_checkpoint_time.py | 59 - .../alerts/alert_datanode_unmounted_data_dir.py | 59 - .../package/alerts/alert_nfs_gateway_process.py | 59 - .../package/alerts/alert_snamenode_process.py | 59 - .../package/alerts/alert_upgrade_finalized.py | 59 - .../services/HDFS/package/scripts/datanode.py | 42 - .../HDFS/package/scripts/hdfs_client.py | 38 - .../HDFS/package/scripts/journalnode.py | 42 - .../services/HDFS/package/scripts/namenode.py | 58 - .../services/HDFS/package/scripts/nfsgateway.py | 42 - .../HDFS/package/scripts/service_check.py | 30 - .../services/HDFS/package/scripts/snamenode.py | 42 - .../services/HDFS/package/scripts/zkfc_slave.py | 38 - .../services/HDFS/quicklinks/quicklinks.json | 76 - .../PERF/1.0/services/HDFS/themes/theme.json | 179 - .../stacks/PERF/1.0/services/HDFS/widgets.json | 649 -- .../PERF/1.0/services/SLEEPY/kerberos.json | 2 +- .../PERF/1.0/services/SLEEPY/metainfo.xml | 1 + .../stacks/PERF/1.0/services/SNOW/kerberos.json | 2 +- .../stacks/PERF/1.0/services/SNOW/metainfo.xml | 1 + .../PERF/1.0/services/YARN/YARN_metrics.json | 3486 ------- .../PERF/1.0/services/YARN/YARN_widgets.json | 611 -- .../stacks/PERF/1.0/services/YARN/alerts.json | 77 - .../YARN/configuration-mapred/mapred-env.xml | 50 - .../YARN/configuration-mapred/mapred-site.xml | 134 - .../YARN/configuration/capacity-scheduler.xml | 69 - .../YARN/configuration/ranger-yarn-audit.xml | 121 - .../ranger-yarn-plugin-properties.xml | 82 - .../configuration/ranger-yarn-policymgr-ssl.xml | 66 - .../YARN/configuration/ranger-yarn-security.xml | 58 - .../YARN/configuration/yarn-alert-config.xml | 80 - .../services/YARN/configuration/yarn-env.xml | 201 - .../services/YARN/configuration/yarn-log4j.xml | 103 - .../services/YARN/configuration/yarn-site.xml | 796 -- .../stacks/PERF/1.0/services/YARN/kerberos.json | 278 - .../stacks/PERF/1.0/services/YARN/metainfo.xml | 355 - .../package/alerts/alert_history_process.py | 59 - .../package/alerts/alert_nodemanager_health.py | 59 - .../alerts/alert_resourcemanager_process.py | 59 - .../package/alerts/alert_timeline_process.py | 59 - .../scripts/application_timeline_server.py | 42 - .../YARN/package/scripts/historyserver.py | 42 - .../package/scripts/mapred_service_check.py | 30 - .../YARN/package/scripts/mapreduce2_client.py | 38 - .../YARN/package/scripts/nodemanager.py | 42 - .../YARN/package/scripts/resourcemanager.py | 48 - .../YARN/package/scripts/service_check.py | 30 - .../YARN/package/scripts/yarn_client.py | 38 - .../YARN/quicklinks-mapred/quicklinks.json | 76 - .../services/YARN/quicklinks/quicklinks.json | 76 - .../1.0/services/YARN/themes-mapred/theme.json | 132 - .../PERF/1.0/services/YARN/themes/theme.json | 250 - .../PERF/1.0/services/ZOOKEEPER/alerts.json | 20 - .../ZOOKEEPER/configuration/zk-alert-config.xml | 80 - .../PERF/1.0/services/ZOOKEEPER/kerberos.json | 39 - .../PERF/1.0/services/ZOOKEEPER/metainfo.xml | 54 - .../package/alerts/alert_zk_server_process.py | 59 - .../ZOOKEEPER/package/scripts/service_check.py | 30 - .../package/scripts/zookeeper_client.py | 38 - .../package/scripts/zookeeper_server.py | 42 - .../stacks/PERF/1.0/upgrades/config-upgrade.xml | 34 + .../1.0/upgrades/nonrolling-upgrade-2.0.xml | 443 + .../stacks/PERF/1.0/upgrades/upgrade-2.0.xml | 331 + .../main/resources/stacks/PERF/2.0/metainfo.xml | 23 + .../stacks/PERF/2.0/repos/repoinfo.xml | 33 + .../resources/stacks/PERF/PythonExecutor.sed | 5 + .../resources/stacks/PERF/install_packages.sed | 11 + contrib/utils/perf/deploy-gce-perf-cluster.py | 4 + 220 files changed, 34511 insertions(+), 33289 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py index d75fc0c..dfe361a 100644 --- a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py +++ b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py @@ -91,10 +91,9 @@ class PythonExecutor(object): recreated or appended. The structured out file, however, is preserved during multiple invocations that use the same file. """ - pythonCommand = self.python_command(script, script_params) logger.debug("Running command " + pprint.pformat(pythonCommand)) - + if handle is None: tmpout, tmperr = self.open_subprocess_files(tmpoutfile, tmperrfile, override_output_files, backup_log_files) @@ -111,10 +110,10 @@ class PythonExecutor(object): self.event.set() thread.join() result = self.prepare_process_result(process.returncode, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=timeout) - + if log_info_on_failure and result['exitcode']: self.on_failure(pythonCommand, result) - + return result else: holder = Holder(pythonCommand, tmpoutfile, tmperrfile, tmpstructedoutfile, handle) @@ -122,7 +121,7 @@ class PythonExecutor(object): background = BackgroundThread(holder, self) background.start() return {"exitcode": 777} - + def on_failure(self, pythonCommand, result): """ Log some useful information after task failure. @@ -134,11 +133,11 @@ class PythonExecutor(object): cmd_list = ["ps faux", "netstat -tulpn"] shell_runner = shellRunner() - + for cmd in cmd_list: ret = shell_runner.run(cmd) logger.info("Command '{0}' returned {1}. {2}{3}".format(cmd, ret["exitCode"], ret["error"], ret["output"])) - + def prepare_process_result(self, returncode, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=None): out, error, structured_out = self.read_result_from_files(tmpoutfile, tmperrfile, tmpstructedoutfile) @@ -166,7 +165,7 @@ class PythonExecutor(object): else: structured_out = {} return out, error, structured_out - + def preexec_fn(self): os.setpgid(0, 0) @@ -197,14 +196,14 @@ class PythonExecutor(object): def condenseOutput(self, stdout, stderr, retcode, structured_out): log_lines_count = self.config.get('heartbeat', 'log_lines_count') - + result = { "exitcode": retcode, "stdout": self.grep.tail(stdout, log_lines_count) if log_lines_count else stdout, "stderr": self.grep.tail(stderr, log_lines_count) if log_lines_count else stderr, "structuredOut" : structured_out } - + return result def python_watchdog_func(self, python, timeout): http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-common/src/main/python/resource_management/libraries/script/dummy.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/dummy.py b/ambari-common/src/main/python/resource_management/libraries/script/dummy.py index 3dcece5..ad5f2a6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/dummy.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/dummy.py @@ -23,16 +23,21 @@ __all__ = ["Dummy"] # Python Imports import os +import re # Local Imports from resource_management.libraries.script.script import Script from resource_management.core.resources.system import Directory, File, Execute from ambari_commons.constants import AMBARI_SUDO_BINARY from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature class Dummy(Script): """ @@ -69,12 +74,25 @@ class Dummy(Script): def install(self, env): print "Install" self.prepare() + component_name = self.get_component_name() + repo_info = str(default("/hostLevelParams/repo_info", "1.1.1.1-1")) + matches = re.findall(r"([\d\.]+\-\d+)", repo_info) + version = matches[0] if matches and len(matches) > 0 else "1.1.1.1-1" + + from resource_management.libraries.functions import stack_tools + (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME) + command = 'ambari-python-wrap {0} install {1}'.format(stack_selector_path, version) + Execute(command) + + if component_name: + conf_select.select("PERF", component_name, version) + stack_select.select(component_name, version) def configure(self, env): print "Configure" self.prepare() - def start(self, env): + def start(self, env, upgrade_type=None): print "Start" self.prepare() @@ -101,7 +119,7 @@ class Dummy(Script): content="" ) - def stop(self, env): + def stop(self, env, upgrade_type=None): print "Stop" self.prepare() @@ -115,3 +133,10 @@ class Dummy(Script): if not os.path.isfile(self.pid_file): raise ComponentIsNotRunning() + + def get_component_name(self): + """ + To be overridden by subclasses. + Returns a string with the component name used in selecting the version. + """ + pass \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py index d70e87f..b7fcc51 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py @@ -61,6 +61,8 @@ class UpgradeSetAll(Script): stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) + if code != 0: + raise Exception("Command '{0}' exit code is nonzero".format(cmd)) if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/conf-select.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/conf-select.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/conf-select.py new file mode 100644 index 0000000..67d0b71 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/conf-select.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import sys + + + +# main method to parse arguments from user and start work +def do_work(args): + pass + + +def main(): + + if len(sys.argv) <= 1: + sys.exit(-1) + + args = sys.argv[1:] + do_work(args) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/distro-select.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/distro-select.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/distro-select.py new file mode 100644 index 0000000..6d9a814 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/distro-select.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import sys +import os + +AMBARI_AGENT_HOST_DIR = "AMBARI_AGENT_HOST_DIR" + +SYMLINKS_TXT = "symlinks.txt" +VERSIONS_TXT = "versions.txt" + +# main method to parse arguments from user and start work +def main(): + if len(sys.argv) <= 1: + sys.exit(-1) + + args = sys.argv[1:] + + do_work(args) + + +def extrakt_var_from_pythonpath(name): + PATH = os.environ['PATH'] + paths = PATH.split(':') + var = '' + for item in paths: + if item.startswith(name): + var = item.replace(name, '') + break + return var + + +def print_versions(args): + dest = versions_file_destination() + + with open(dest, 'r') as f: + for line in f: + print line + + +def versions_file_destination(): + agent_host_dir = extrakt_var_from_pythonpath(AMBARI_AGENT_HOST_DIR) + dest = os.path.join(agent_host_dir, VERSIONS_TXT) + if not os.path.exists(dest): + open(dest, 'w').close() + return dest + + +def print_status(args): + dest = symlinks_file_destination() + + with open(dest, 'r') as f: + if len(args) >= 2: + for line in f: + if args[1] in line: + print line + pass + + for line in f: + print line + + +def set_version(args): + dest = symlinks_file_destination() + + line_template = "{0} - {1}\n" + result = "" + with open(dest, 'r') as f: + + if len(args) >= 3: + if args[1] != "all": + seted = False + for line in f: + if args[1] in line: + compinfo = str.split(line) + result += line_template.format(compinfo[0], args[2]) + seted = True + else: + result += line + if seted != True: + result += line_template.format(args[1], args[2]) + else: + for line in f: + compinfo = str.split(line) + result += line_template.format(compinfo[0], args[2]) + + with open(dest, 'w') as f: + f.write(result) + + +def symlinks_file_destination(): + agent_host_dir = extrakt_var_from_pythonpath(AMBARI_AGENT_HOST_DIR) + dest = os.path.join(agent_host_dir, SYMLINKS_TXT) + if not os.path.exists(dest): + open(dest, 'w').close() + return dest + + +def install_version(args): + dest = versions_file_destination() + installed = False + with open(dest, 'r') as f: + for line in f: + if args[1] in line: + installed = True + break + if not installed: + with open(dest, 'a') as f: + if args[1]: + f.write(args[1] + "\n") + +def do_work(args): + """ + Check that all required args are passed in. If so, perform required action. + :param args: + """ + if not args[0] or args[0] == "status": + print_status(args) + elif args[0] == "versions": + print_versions(args) + elif args[0] == "set": + set_version(args) + elif args[0] == "install": + install_version(args) + + + +if __name__ == "__main__": + main() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/hook.py index 9eca9ec..175f39a 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/hook.py +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/hook.py @@ -16,13 +16,44 @@ See the License for the specific language governing permissions and limitations under the License. """ - +import os +from resource_management.core.resources.system import Directory, File, Execute from resource_management.libraries.script import Hook +AMBARI_AGENT_CACHE_DIR = 'AMBARI_AGENT_CACHE_DIR' + +BEFORE_INSTALL_SCRIPTS = "hooks/before-INSTALL/scripts" +STACK = "PERF/1.0" +STACKS = "stacks" +DISTRO_SELECT_PY = os.path.join(STACKS, STACK, BEFORE_INSTALL_SCRIPTS, "distro-select.py") +CONF_SELECT_PY = os.path.join(STACKS, STACK, BEFORE_INSTALL_SCRIPTS, "conf-select.py") +DISTRO_SELECT_DEST = "/usr/bin/distro-select" +CONF_SELECT_DEST = "/usr/bin/conf-select" + class BeforeInstallHook(Hook): def hook(self, env): print "Before Install Hook" + cache_dir = self.extrakt_var_from_pythonpath(AMBARI_AGENT_CACHE_DIR) + conf_select = os.path.join(cache_dir, CONF_SELECT_PY) + dist_select = os.path.join(cache_dir, DISTRO_SELECT_PY) + if not os.path.exists(CONF_SELECT_DEST): + Execute("cp %s %s" % (conf_select, CONF_SELECT_DEST), user="root") + Execute("chmod a+x %s" % (CONF_SELECT_DEST), user="root") + if not os.path.exists(DISTRO_SELECT_DEST): + Execute("cp %s %s" % (dist_select, DISTRO_SELECT_DEST), user="root") + Execute("chmod a+x %s" % (DISTRO_SELECT_DEST), user="root") + + def extrakt_var_from_pythonpath(self, name): + + PATH = os.environ['PATH'] + paths = PATH.split(':') + var = '' + for item in paths: + if item.startswith(name): + var = item.replace(name, '') + break + return var if __name__ == "__main__": BeforeInstallHook().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/params.py new file mode 100644 index 0000000..6eddfdf --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/hooks/before-INSTALL/scripts/params.py @@ -0,0 +1,23 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions import default, format + + +version = default("/commandParams/version", None) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/metainfo.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/metainfo.xml index 45a63e5..90556f1 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/metainfo.xml @@ -16,7 +16,7 @@ limitations under the License. --> <metainfo> - <versions> - <active>false</active> - </versions> + <versions> + <active>false</active> + </versions> </metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/repos/repoinfo.xml index c5137fb..0895fab 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/repos/repoinfo.xml @@ -18,10 +18,16 @@ <reposinfo> <os family="redhat6"> <repo> - <baseurl>http://foo</baseurl> + <baseurl>http://foo-1.0.1.0-1</baseurl> <repoid>PERF-1.0</repoid> <reponame>PERF</reponame> <unique>true</unique> </repo> + <repo> + <baseurl>http://foo2</baseurl> + <repoid>PERF-UTILS-1.0</repoid> + <reponame>PERF-UTILS</reponame> + <unique>false</unique> + </repo> </os> </reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/alerts.json b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/alerts.json new file mode 100644 index 0000000..f864f45 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/alerts.json @@ -0,0 +1,35 @@ +{ + "FAKEHBASE": { + + "FAKEHBASE_MASTER": [ + { + "name": "hbase_master_process", + "label": "FAKEHBase Master Process", + "description": "This alert is triggered if the HBase master processes cannot be confirmed to be up and listening on the network for the configured critical threshold, given in seconds.", + "interval": 1, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "PERF/1.0/services/FAKEHBASE/package/alerts/hbase_master_process.py", + "parameters": [] + } + } + ], + "FAKEHBASE_REGIONSERVER": [ + { + "name": "hbase_regionserver_process", + "label": "HBase FAKERegionServer Process", + "description": "This host-level alert is triggered if the FAKERegionServer processes cannot be confirmed to be up and listening on the network for the configured critical threshold, given in seconds.", + "interval": 1, + "scope": "HOST", + "enabled": true, + "source": { + "type": "SCRIPT", + "path": "PERF/1.0/services/FAKEHBASE/package/alerts/hbase_regionserver_process.py", + "parameters": [] + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-alert-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-alert-config.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-alert-config.xml new file mode 100644 index 0000000..392eea7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-alert-config.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> + + <property> + <name>alert.behavior.type</name> + <value>percentage</value> + <description> + This property describes type of alert behaviour. + There are three types percentage, timeout, flip. + </description> + <on-ambari-upgrade add="false"/> + </property> + + + <property> + <name>alert.success.percentage</name> + <value>100</value> + <description> + This property will be actual only when alert.behaviour.type + set to "percentage". Here you should set percent of successful + alert checks. + </description> + <on-ambari-upgrade add="false"/> + </property> + + + <property> + <name>alert.timeout.return.value</name> + <value>false</value> + <description> + This property will be actual only when alert.behaviour.type + set to "timeout". Here you should set result which alert will + return after timeout, false|true|none. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>alert.timeout.secs</name> + <value>120</value> + <description> + This property will be actual only when alert.behaviour.type + set to "timeout". Here you should set number of seconds for + alert to sleep. + </description> + <on-ambari-upgrade add="false"/> + </property> + + + <property> + <name>alert.flip.interval.mins</name> + <value>3</value> + <description> + This property will be actual only when alert.behaviour.type + set to "flip". Here you should set number of minutes at which + the alert should flip from true|false. + </description> + <on-ambari-upgrade add="false"/> + </property> + + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-env.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-env.xml new file mode 100644 index 0000000..91fed20 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-env.xml @@ -0,0 +1,292 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- These properties exist in common services. --> + <property> + <name>hbase_log_dir</name> + <value>/var/log/hbase</value> + <display-name>HBase Log Dir Prefix</display-name> + <description>Log Directories for HBase.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_regionserver_heapsize</name> + <value>4096</value> + <description>Maximum amount of memory each HBase FAKERegionServer can use.</description> + <display-name>HBase FAKERegionServer Maximum Memory</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>6554</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_master_heapsize</name> + <value>4096</value> + <description>Maximum amount of memory each FAKEHBase Master can use.</description> + <display-name>FAKEHBase Master Maximum Memory</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>16384</maximum> + <unit>MB</unit> + <increment-step>256</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_user_nofile_limit</name> + <value>32000</value> + <description>Max open files limit setting for FAKEHBASE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_user_nproc_limit</name> + <value>16000</value> + <description>Max number of processes limit setting for FAKEHBASE user.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_java_io_tmpdir</name> + <value>/tmp</value> + <description>Used in hbase-env.sh as FAKEHBASE_OPTS=-Djava.io.tmpdir=java_io_tmpdir</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_principal_name</name> + <description>HBase principal name</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_user_keytab</name> + <description>HBase keytab path</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_regionserver_shutdown_timeout</name> + <value>30</value> + <display-name>HBase FAKERegionServer shutdown timeout</display-name> + <description> + After this number of seconds waiting for graceful stop of FAKEHBase Master it will be forced to exit with SIGKILL. + The timeout is introduced because there is a known bug when from time to time HBase FAKERegionServer hangs forever on stop if NN safemode is on. + </description> + <value-attributes> + <type>int</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These properties exist in HDP 2.2 and higher. --> + <property> + <name>hbase_log_dir</name> + <value>/var/log/hbase</value> + <display-name>HBase Log Dir Prefix</display-name> + <description>Log Directories for HBase.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_pid_dir</name> + <value>/var/run/hbase</value> + <display-name>HBase PID Dir</display-name> + <description>Pid Directory for HBase.</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_regionserver_xmn_max</name> + <value>512</value> + <description> + Sets the upper bound on HBase FAKEFAKERegionServers' young generation size. + This value is used in case the young generation size (-Xmn) calculated based on the max heapsize (hbase_regionserver_heapsize) + and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value. + </description> + <display-name>FAKEFAKERegionServers maximum value for -Xmn</display-name> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_regionserver_xmn_ratio</name> + <value>0.2</value> + <display-name>FAKEFAKERegionServers -Xmn in -Xmx ratio</display-name> + <description>Percentage of max heap size (-Xmx) which used for young generation heap (-Xmn).</description> + <value-attributes> + <type>float</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_user</name> + <display-name>HBase User</display-name> + <value>hbase</value> + <property-type>USER</property-type> + <description>HBase User Name.</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase_max_direct_memory_size</name> + <value/> + <display-name>HBase off-heap MaxDirectMemorySize</display-name> + <description>If not empty, adds '-XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m' to FAKEHBASE_REGIONSERVER_OPTS.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>phoenix_sql_enabled</name> + <value>false</value> + <description>Enable Phoenix SQL</description> + <display-name>Enable Phoenix</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>true</value> + <label>Enabled</label> + </entry> + <entry> + <value>false</value> + <label>Disabled</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These properties exist in HDP 2.3 and higher. --> + <!-- hbase-env.sh --> + <property> + <name>content</name> + <display-name>hbase-env template</display-name> + <description>This is the jinja template for hbase-env.sh file</description> + <value> +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export FAKEHBASE_CONF_DIR=${FAKEHBASE_CONF_DIR:-{{hbase_conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export FAKEHBASE_CLASSPATH=${FAKEHBASE_CLASSPATH} + + +# The maximum amount of heap to use, in MB. Default is 1000. +# export FAKEHBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export FAKEHBASE_OPTS="$FAKEHBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$FAKEHBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export FAKEHBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size +# export FAKEHBASE_THRIFT_OPTS="$FAKEHBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export FAKEHBASE_FAKEZOOKEEPER_OPTS="$FAKEHBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HFAKEFAKERegionServers will run. $FAKEHBASE_HOME/conf/regionservers by default. +export FAKEHBASE_REGIONSERVERS=${FAKEHBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export FAKEHBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=FAKEHBASE_CONF_DIR" + +# Where log files are stored. $FAKEHBASE_HOME/logs by default. +export FAKEHBASE_LOG_DIR={{log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export FAKEHBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export FAKEHBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export FAKEHBASE_PID_DIR={{pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export FAKEHBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export FAKEHBASE_MANAGES_ZK=false + +{% if java_version < 8 %} +JDK_DEPENDED_OPTS="-XX:PermSize=128m -XX:MaxPermSize=128m" +{% endif %} + +{% if security_enabled %} +export FAKEHBASE_OPTS="$FAKEHBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}} -Djava.io.tmpdir={{java_io_tmpdir}}" +export FAKEHBASE_MASTER_OPTS="$FAKEHBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}} $JDK_DEPENDED_OPTS" +export FAKEHBASE_REGIONSERVER_OPTS="$FAKEHBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}} $JDK_DEPENDED_OPTS" +export PHOENIX_QUERYSERVER_OPTS="$PHOENIX_QUERYSERVER_OPTS -Djava.security.auth.login.config={{queryserver_jaas_config_file}}" +{% else %} +export FAKEHBASE_OPTS="$FAKEHBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.io.tmpdir={{java_io_tmpdir}}" +export FAKEHBASE_MASTER_OPTS="$FAKEHBASE_MASTER_OPTS -Xmx{{master_heapsize}} $JDK_DEPENDED_OPTS" +export FAKEHBASE_REGIONSERVER_OPTS="$FAKEHBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} $JDK_DEPENDED_OPTS" +{% endif %} + +# HBase off-heap MaxDirectMemorySize +export FAKEHBASE_REGIONSERVER_OPTS="$FAKEHBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}" +export FAKEHBASE_MASTER_OPTS="$FAKEHBASE_MASTER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}" + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-log4j.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-log4j.xml new file mode 100644 index 0000000..bad36ee --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-log4j.xml @@ -0,0 +1,146 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>hbase-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=INFO +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.FAKEZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO + + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-logsearch-conf.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-logsearch-conf.xml new file mode 100644 index 0000000..891445d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-logsearch-conf.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>HBase</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>FAKEHBASE_MASTER:hbase_master;FAKEHBASE_REGIONSERVER:hbase_regionserver;FAKEPHOENIX_QUERY_SERVER:hbase_phoenix_server</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"hbase_master", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-master-*.log" + }, + { + "type":"hbase_regionserver", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-regionserver-*.log" + }, + { + "type":"hbase_phoenix_server", + "rowtype":"service", + "path":"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/phoenix-*-server.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_master", + "hbase_regionserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_phoenix_server" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-policy.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-policy.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-policy.xml new file mode 100644 index 0000000..66e00c2 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-policy.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + <description>ACL for HRegionInterface protocol implementations (ie. + clients talking to HFAKEFAKERegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterRegionInterface protocol implementations + (for HFAKEFAKERegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2c362fd0/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-site.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-site.xml new file mode 100644 index 0000000..d8df033 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEHBASE/configuration/hbase-site.xml @@ -0,0 +1,555 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- These properties exist in common services. --> + <property> + <name>hbase.rootdir</name> + <value>hdfs://localhost:8020/apps/hbase/data</value> + <description>The directory shared by region servers and into + which HBase persists. The URL should be 'fully-qualified' + to include the filesystem scheme. For example, to specify the + FAKEHDFS directory '/hbase' where the FAKEHDFS instance's namenode is + running at namenode.example.org on port 9000, set this value to: + hdfs://namenode.example.org:9000/hbase. By default HBase writes + into /tmp. Change this configuration else all data will be lost + on machine restart. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + <description>The mode the cluster will be in. Possible values are + false for standalone mode and true for distributed mode. If + false, startup will run all HBase and FAKEZooKeeper daemons together + in the one JVM. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.tmp.dir</name> + <value>/tmp/hbase-${user.name}</value> + <display-name>HBase tmp directory</display-name> + <description>Temporary directory on the local filesystem. + Change this setting to point to a location more permanent + than '/tmp' (The '/tmp' directory is often cleared on + machine restart). + </description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + <description>Directory on the local filesystem to be used as a local storage + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.master.info.bindAddress</name> + <value>0.0.0.0</value> + <description>The bind address for the FAKEHBase Master web UI + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.regionserver.handler.count</name> + <value>30</value> + <description> + Count of RPC Listener instances spun up on FAKEFAKERegionServers. + Same property is used by the Master for count of master handlers. + </description> + <display-name>Number of Handlers per FAKERegionServer</display-name> + <value-attributes> + <type>int</type> + <minimum>5</minimum> + <maximum>240</maximum> + <increment-step>1</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hregion.majorcompaction</name> + <value>604800000</value> + <description> + Time between major compactions. Set to 0 to disable automatic major compactions. + </description> + <display-name>Major Compaction Interval</display-name> + <value-attributes> + <type>int</type> + <minimum>0</minimum> + <maximum>2592000000</maximum> + <unit>milliseconds</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>4</value> + <description> + Block updates if a memstore's size spikes this many times above the size that would cause it to be flushed. + Useful to prevent runaway memstores during a sudden spike in update traffic. + </description> + <display-name>Per-Column Family Memstore Block Multiplier</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>2</value> + </entry> + <entry> + <value>4</value> + </entry> + <entry> + <value>8</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>134217728</value> + <description> + The size of an individual memstore. Each column familiy within each region is allocated its own memstore. + </description> + <display-name>Memstore Flush Size</display-name> + <value-attributes> + <type>int</type> + <minimum>33554432</minimum> + <maximum>268435456</maximum> + <increment-step>1048576</increment-step> + <unit>B</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hregion.memstore.mslab.enabled</name> + <value>true</value> + <description> + Enables the MemStore-Local Allocation Buffer, + a feature which works to prevent heap fragmentation under + heavy write loads. This can reduce the frequency of stop-the-world + GC pauses on large heaps. + </description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hregion.max.filesize</name> + <value>10737418240</value> + <description> + Maximum HFile size. If the sum of the sizes of a region's HFiles has grown to exceed this + value, the region is split in two. + </description> + <display-name>Maximum Region File Size</display-name> + <value-attributes> + <type>int</type> + <minimum>1073741824</minimum> + <maximum>107374182400</maximum> + <unit>B</unit> + <increment-step>1073741824</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.client.scanner.caching</name> + <value>100</value> + <description>Number of rows that will be fetched when calling next + on a scanner if it is not served from (local, client) memory. Higher + caching values will enable faster scanners but will eat up more memory + and some calls of next may take longer and longer times when the cache is empty. + Do not set this value such that the time between invocations is greater + than the scanner timeout; i.e. hbase.regionserver.lease.period + </description> + <display-name>Number of Fetched Rows when Scanning from Disk</display-name> + <value-attributes> + <type>int</type> + <minimum>100</minimum> + <maximum>10000</maximum> + <increment-step>100</increment-step> + <unit>rows</unit> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>zookeeper.session.timeout</name> + <value>90000</value> + <description>FAKEZooKeeper session timeout. + FAKEZooKeeper session timeout in milliseconds. It is used in two different ways. + First, this value is used in the ZK client that HBase uses to connect to the ensemble. + It is also used by HBase when it starts a ZK server and it is passed as the 'maxSessionTimeout'. See + http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions. + For example, if a HBase region server connects to a ZK ensemble that's also managed by HBase, then the + session timeout will be the one specified by this configuration. But, a region server that connects + to an ensemble managed with a different configuration will be subjected that ensemble's maxSessionTimeout. So, + even though HBase might propose using 90 seconds, the ensemble can have a max timeout lower than this and + it will take precedence. + </description> + <display-name>Zookeeper Session Timeout</display-name> + <value-attributes> + <type>int</type> + <minimum>10000</minimum> + <maximum>180000</maximum> + <unit>milliseconds</unit> + <increment-step>10000</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>1048576</value> + <description> + Specifies the combined maximum allowed size of a KeyValue + instance. This is to set an upper boundary for a single entry saved in a + storage file. Since they cannot be split it helps avoiding that a region + cannot be split any further because the data is too large. It seems wise + to set this to a fraction of the maximum region size. Setting it to zero + or less disables the check. + </description> + <display-name>Maximum Record Size</display-name> + <value-attributes> + <type>int</type> + <minimum>1048576</minimum> + <maximum>31457280</maximum> + <unit>B</unit> + <increment-step>262144</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hstore.compactionThreshold</name> + <value>3</value> + <description> + The maximum number of StoreFiles which will be selected for a single minor + compaction, regardless of the number of eligible StoreFiles. Effectively, the value of + hbase.hstore.compaction.max controls the length of time it takes a single compaction to + complete. Setting it larger means that more StoreFiles are included in a compaction. For most + cases, the default value is appropriate. + </description> + <display-name>Maximum Store Files before Minor Compaction</display-name> + <value-attributes> + <type>int</type> + <entries> + <entry> + <value>2</value> + </entry> + <entry> + <value>3</value> + </entry> + <entry> + <value>4</value> + </entry> + </entries> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hstore.flush.retries.number</name> + <value>120</value> + <description> + The number of times the region flush operation will be retried. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <display-name>hstore blocking storefiles</display-name> + <value>10</value> + <description> + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + </description> + <value-attributes> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hfile.block.cache.size</name> + <value>0.40</value> + <description>Percentage of FAKERegionServer memory to allocate to read buffers.</description> + <display-name>% of FAKERegionServer Allocated to Read Buffers</display-name> + <value-attributes> + <type>float</type> + <minimum>0</minimum> + <maximum>0.8</maximum> + <increment-step>0.01</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <!-- Additional configuration specific to HBase security --> + <property> + <name>hbase.superuser</name> + <value>hbase</value> + <description>List of users or groups (comma-separated), who are allowed + full privileges, regardless of stored ACLs, across the cluster. + Only used when HBase security is enabled. + </description> + <depends-on> + <property> + <type>hbase-env</type> + <name>hbase_user</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.security.authentication</name> + <value>simple</value> + <description> + Select Simple or Kerberos authentication. Note: Kerberos must be set up before the Kerberos option will take effect. + </description> + <display-name>Enable Authentication</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <label>Simple</label> + <value>simple</value> + </entry> + <entry> + <label>Kerberos</label> + <value>kerberos</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.security.authorization</name> + <value>false</value> + <description> Set Authorization Method.</description> + <display-name>Enable Authorization</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>true</value> + <label>Native</label> + </entry> + <entry> + <value>false</value> + <label>Off</label> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value> + <description>A comma-separated list of Coprocessors that are loaded by + default on all tables. For any override coprocessor method, these classes + will be called in order. After implementing your own Coprocessor, just put + it in HBase's classpath and add the fully qualified class name here. + A coprocessor can also be loaded on demand by setting HTableDescriptor. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>hbase-site</type> + <name>hbase.security.authorization</name> + </property> + <property> + <type>hbase-site</type> + <name>hbase.security.authentication</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.coprocessor.master.classes</name> + <value/> + <description>A comma-separated list of + org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are + loaded by default on the active HMaster process. For any implemented + coprocessor methods, the listed classes will be called in order. After + implementing your own MasterObserver, just put it in HBase's classpath + and add the fully qualified class name here. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>hbase-site</type> + <name>hbase.security.authorization</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + <description>Property from FAKEZooKeeper's config zoo.cfg. + The port at which the clients will connect. + </description> + <on-ambari-upgrade add="false"/> + </property> + + <!-- End of properties used to generate FAKEZooKeeper host:port quorum list. --> + <property> + <name>hbase.zookeeper.useMulti</name> + <value>true</value> + <description>Instructs HBase to make use of FAKEZooKeeper's multi-update functionality. + This allows certain FAKEZooKeeper operations to complete more quickly and prevents some issues + with rare Replication failure scenarios (see the release note of FAKEHBASE-2611 for an example).· + IMPORTANT: only set this to true if all FAKEZooKeeper servers in the cluster are on version 3.4+ + and will not be downgraded. FAKEZooKeeper versions before 3.4 do not support multi-update and will + not fail gracefully if multi-update is invoked (see FAKEZOOKEEPER-1495). + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase-unsecure</value> + <description>Root ZNode for HBase in FAKEZooKeeper. All of HBase's FAKEZooKeeper + files that are configured with a relative path will go under this node. + By default, all of HBase's FAKEZooKeeper file path are configured with a + relative path, so they will all go under this directory unless changed. + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.client.retries.number</name> + <value>35</value> + <description>Maximum retries. Used as maximum for all retryable + operations such as the getting of a cell's value, starting a row update, + etc. Retry interval is a rough function based on hbase.client.pause. At + first we retry at this interval but then with backoff, we pretty quickly reach + retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup + ramps up. Change this setting and hbase.client.pause to suit your workload.</description> + <display-name>Maximum Client Retries</display-name> + <value-attributes> + <type>int</type> + <minimum>5</minimum> + <maximum>50</maximum> + <increment-step>1</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.rpc.timeout</name> + <value>90000</value> + <description> + This is for the RPC layer to define how long HBase client applications + take for a remote call to time out. It uses pings to check connections + but will eventually throw a TimeoutException. + </description> + <display-name>HBase RPC Timeout</display-name> + <value-attributes> + <type>int</type> + <minimum>10000</minimum> + <maximum>180000</maximum> + <unit>milliseconds</unit> + <increment-step>10000</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.defaults.for.version.skip</name> + <value>true</value> + <description>Disables version verification.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>phoenix.query.timeoutMs</name> + <value>60000</value> + <description>Number of milliseconds after which a Phoenix query will timeout on the client.</description> + <display-name>Phoenix Query Timeout</display-name> + <value-attributes> + <type>int</type> + <minimum>30000</minimum> + <maximum>180000</maximum> + <unit>milliseconds</unit> + <increment-step>10000</increment-step> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + <description>Path to domain socket.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.rpc.protection</name> + <value>authentication</value> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These properties exist in HDP 2.3 and higher. --> + <property> + <name>hbase.master.port</name> + <value>16000</value> + <display-name>FAKEHBase Master Port</display-name> + <description>The port the FAKEHBase Master should bind to.</description> + <value-attributes> + <overridable>false</overridable> + <type>int</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.master.info.port</name> + <value>16010</value> + <description>The port for the FAKEHBase Master web UI.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.regionserver.port</name> + <value>16020</value> + <description>The port the HBase FAKERegionServer binds to.</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hbase.regionserver.info.port</name> + <value>16030</value> + <description>The port for the HBase FAKERegionServer web UI.</description> + <on-ambari-upgrade add="false"/> + </property> + + + <!-- These properties exist in HDP 2.5 and higher. --> + <property> + <name>hbase.master.ui.readonly</name> + <value>false</value> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>zookeeper.recovery.retry</name> + <value>6</value> + <on-ambari-upgrade add="false"/> + </property> +</configuration>
