SLIDER-1107 Generate app configuration files in AM -- squash merge from feature/SLIDER-1107_AM_config_generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/424b2a40 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/424b2a40 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/424b2a40 Branch: refs/heads/develop Commit: 424b2a4062b96ada5eeb46b9d177619955af86a4 Parents: 094f079 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Aug 3 16:23:19 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Wed Aug 3 16:41:11 2016 -0700 ---------------------------------------------------------------------- app-packages/hbase-nopkg/README.md | 56 ++++ app-packages/hbase-nopkg/appConfig-default.json | 112 ++++++++ .../hbase-nopkg/appConfig-secured-default.json | 128 ++++++++++ .../clientInstallConfig-default.json | 6 + app-packages/hbase-nopkg/metainfo-secured.xml | 255 +++++++++++++++++++ app-packages/hbase-nopkg/metainfo.xml | 225 ++++++++++++++++ app-packages/hbase-nopkg/pom.xml | 92 +++++++ app-packages/hbase-nopkg/resources-default.json | 47 ++++ ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 +++++ ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 +++++ .../hadoop-metrics2-hbase.properties-MASTER.j2 | 47 ++++ .../hadoop-metrics2-hbase.properties-RS.j2 | 47 ++++ .../resources/hbase-env-client.sh.j2 | 49 ++++ .../resources/hbase-env-secured.sh.j2 | 63 +++++ .../hbase-nopkg/resources/hbase-env.sh.j2 | 59 +++++ .../resources/hbase-log4j.properties.j2 | 110 ++++++++ .../resources/hbase_client_jaas.conf.j2 | 22 ++ .../resources/hbase_master_jaas.conf.j2 | 25 ++ .../resources/hbase_regionserver_jaas.conf.j2 | 25 ++ app-packages/hbase-nopkg/src/assembly/hbase.xml | 89 +++++++ pom.xml | 7 + .../src/main/python/agent/ActionQueue.py | 2 +- .../python/scripts/shell_cmd/basic_installer.py | 11 +- slider-core/pom.xml | 6 + .../org/apache/slider/client/ClientUtils.java | 111 ++++++++ .../org/apache/slider/client/SliderClient.java | 204 ++++++++++----- .../apache/slider/client/SliderClientAPI.java | 13 +- .../org/apache/slider/common/SliderKeys.java | 3 +- .../common/params/ActionResourceArgs.java | 68 +++++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 10 + .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 96 ++++++- .../slider/core/conf/ConfTreeOperations.java | 16 +- .../slider/core/launch/AbstractLauncher.java | 21 +- .../core/persist/AppDefinitionPersister.java | 6 +- .../core/registry/docstore/ConfigFormat.java | 4 +- .../core/registry/docstore/ConfigUtils.java | 96 +++++++ .../PublishedConfigurationOutputter.java | 64 +++-- .../providers/AbstractClientProvider.java | 9 +- .../providers/agent/AgentClientProvider.java | 215 +++++++++++----- .../slider/providers/agent/AgentKeys.java | 7 + .../providers/agent/AgentProviderService.java | 255 +++++++++++++++++-- .../application/metadata/AbstractComponent.java | 4 + .../agent/application/metadata/Application.java | 4 + .../agent/application/metadata/Component.java | 12 +- .../agent/application/metadata/Metainfo.java | 13 + .../application/metadata/MetainfoParser.java | 19 +- .../app_packages/test_am_config/appConfig.json | 31 +++ .../app_packages/test_am_config/metainfo.json | 70 +++++ .../app_packages/test_am_config/resources.json | 17 ++ .../test_am_config/resources/test.template | 1 + .../test_am_config/test_archive/testfile | 1 + .../resources_unique_names.json | 18 ++ .../TestPublishedConfigurationOutputter.java | 221 ++++++++++++++++ .../agent/TestAgentClientProvider2.java | 8 +- .../agent/TestAgentProviderService.java | 66 +++-- .../metadata/TestMetainfoParser.java | 13 +- .../agent/application/metadata/metainfo.xml | 21 ++ .../apache/slider/funtest/ResourcePaths.groovy | 5 + .../funtest/misc/AMConfigPublishingIT.groovy | 146 +++++++++++ .../funtest/misc/UniqueComponentNamesIT.groovy | 122 +++++++++ 62 files changed, 3381 insertions(+), 219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/README.md ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/README.md b/app-packages/hbase-nopkg/README.md new file mode 100644 index 0000000..43fd7d4 --- /dev/null +++ b/app-packages/hbase-nopkg/README.md @@ -0,0 +1,56 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Create Slider App Package for HBase + +appConfig-default.json and resources-default.json are not required to be packaged. +These files are included as reference configuration for Slider apps and are suitable +for a one-node cluster. + +To create the app package you will need the HBase tarball and invoke mvn command +with appropriate parameters. It is recommended that pkg.version be set to the + same value as hbase.version. + +Command: + + mvn clean package -Phbase-resources -Dhbase.version=<hbase version> -Dpkg.version=<app package version> + -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available> + +Example: + + mvn clean package -Phbase-resources -Dhbase.version=1.1.4 + -Dpkg.version=1.1.4 -Dpkg.name=hbase-1.1.4-bin.tar.gz + -Dpkg.src=/Users/user1/Downloads + +App package can be found in + + app-packages/hbase-nopkg/target/slider-hbase-resources-1.1.4.zip + +## Verifying the content + +Verify the content using + + zip -Tv slider-hbase-*.zip + +## Sample commands + + unzip slider-hbase-resources-1.1.4.zip + slider resource --install --resource resources --destdir hbase + slider create hbase --template appConfig-default.json --resources resources-default.json --metainfo metainfo.xml + slider client --install --dest client_install_dir --name hbase --config clientInstallConfig-default.json + + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json new file mode 100644 index 0000000..1095efd --- /dev/null +++ b/app-packages/hbase-nopkg/appConfig-default.json @@ -0,0 +1,112 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "am.config.generation": "true", + "create.default.zookeeper.node": "true", + "java_home": "/usr/jdk64/jdk1.8.0_60", + "system_configs": "core-site", + + "site.global.app_user": "yarn", + "site.global.app_version": "${hbase.version}", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/packages/hbase-${hbase.version}", + + "site.global.pid_file": "${@//site/global/app_pid_dir}/hbase-${@//site/global/app_user}-${@//site/global/hbase_component}.pid", + "site.global.daemon_cmd": "env HBASE_IDENT_STRING=${@//site/global/app_user} ${@//site/global/app_root}/bin/hbase-daemon.sh --config ${@//site/global/app_conf_dir}", + + "site.global.hbase_instance_name": "instancename", + "site.global.hbase_root_password": "secret", + "site.global.user_group": "hadoop", + + "site.global.thrift_port": "0", + "site.global.thrift2_port": "0", + "site.global.rest_port": "0", + + "site.hbase-env.template.file": "hbase-env.sh.j2", + "site.hbase-env.java64_home": "${JAVA_HOME}", + "site.hbase-env.conf_dir": "${@//site/global/app_conf_dir}", + "site.hbase-env.log_dir": "${@//site/global/app_log_dir}", + "site.hbase-env.pid_dir": "${@//site/global/app_pid_dir}", + "site.hbase-env.master_heapsize": "1024m", + "site.hbase-env.regionserver_heapsize": "1024m", + "site.hbase-env.regionserver_xmn_size": "512m", + "site.hbase-env.hbase_opts": "", + + "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data", + "site.hbase-site.hbase.cluster.distributed": "true", + "site.hbase-site.hbase.superuser": "${USER_NAME}", + "site.hbase-site.hbase.tmp.dir": "work/app/tmp", + "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local", + "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}", + "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}", + "site.hbase-site.hbase.regionserver.info.port": "0", + "site.hbase-site.hbase.bulkload.staging.dir": "/user/${USER_NAME}/hbase-staging", + "site.hbase-site.hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint", + "site.hbase-site.hbase.master.info.bindAddress": "0.0.0.0", + "site.hbase-site.hbase.master.info.port": "16010", + "site.hbase-site.hbase.regionserver.port": "0", + "site.hbase-site.hbase.master.port": "0", + "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4", + "site.hbase-site.hbase.regionserver.handler.count": "60", + "site.hbase-site.hbase.hregion.majorcompaction": "86400000", + "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38", + "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2", + "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728", + "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true", + "site.hbase-site.hbase.hregion.max.filesize": "10737418240", + "site.hbase-site.hbase.client.scanner.caching": "100", + "site.hbase-site.zookeeper.session.timeout": "30000", + "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760", + "site.hbase-site.hbase.hstore.compactionThreshold": "3", + "site.hbase-site.hbase.hstore.flush.retries.number": "120", + "site.hbase-site.hbase.hstore.blockingStoreFiles": "10", + "site.hbase-site.hfile.block.cache.size": "0.40", + "site.hbase-site.hbase.zookeeper.property.clientPort": "2181", + "site.hbase-site.hbase.zookeeper.useMulti": "true", + "site.hbase-site.hbase.defaults.for.version.skip": "true", + "site.hbase-site.dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket", + + "site.hbase-site.hbase.security.authentication": "simple", + "site.hbase-site.hbase.security.authorization": "false", + "site.hbase-site.hbase.coprocessor.region.classes": "", + "site.hbase-site.hbase.coprocessor.master.classes": "", + + "site.hbase-policy.security.client.protocol.acl": "*", + "site.hbase-policy.security.admin.protocol.acl": "*", + "site.hbase-policy.security.masterregion.protocol.acl": "*", + + "site.hbase-log4j.template.file": "hbase-log4j.properties.j2", + + "site.hbase-metrics2.template.file": "hadoop-metrics2-hbase.properties-RS.j2", + "site.hbase-metrics2.app_name": "${CLUSTER_NAME}", + "site.hbase-metrics2.metric_collector_host": "${NN_HOST}", + "site.hbase-metrics2.metric_collector_port": "6188", + "site.hbase-metrics2.metric_collector_lib": "" + }, + "components": { + "slider-appmaster": { + "jvm.heapsize": "1024M" + }, + "HBASE_MASTER": { + "site.global.hbase_component": "master", + "site.hbase-metrics2.template.file": "hadoop-metrics2-hbase.properties-MASTER.j2" + }, + "HBASE_REGIONSERVER": { + "site.global.hbase_component": "regionserver" + }, + "HBASE_REST": { + "site.global.hbase_component": "rest" + }, + "HBASE_THRIFT": { + "site.global.hbase_component": "thrift" + }, + "HBASE_THRIFT2": { + "site.global.hbase_component": "thrift2" + }, + "HBASE_CLIENT": { + "site.hbase-env.template.file": "hbase-env-client.sh.j2", + "site.hbase-site.instance": "${CLUSTER_NAME}" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/appConfig-secured-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/appConfig-secured-default.json b/app-packages/hbase-nopkg/appConfig-secured-default.json new file mode 100644 index 0000000..ac6e45b --- /dev/null +++ b/app-packages/hbase-nopkg/appConfig-secured-default.json @@ -0,0 +1,128 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "am.config.generation": "true", + "create.default.zookeeper.node": "true", + "java_home": "/usr/jdk64/jdk1.8.0_60", + "system_configs": "core-site,hdfs-site", + + "site.global.app_user": "${USER_NAME}", + "site.global.app_version": "${hbase.version}", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/packages/hbase-${hbase.version}", + + "site.global.pid_file": "${@//site/global/app_pid_dir}/hbase-${@//site/global/app_user}-${@//site/global/hbase_component}.pid", + "site.global.daemon_cmd": "env HBASE_IDENT_STRING=${@//site/global/app_user} ${@//site/global/app_root}/bin/hbase-daemon.sh --config ${@//site/global/app_conf_dir}", + + "site.global.hbase_instance_name": "instancename", + "site.global.hbase_root_password": "secret", + "site.global.user_group": "hadoop", + + "site.global.thrift_port": "0", + "site.global.thrift2_port": "0", + "site.global.rest_port": "0", + + "site.hbase-env.template.file": "hbase-env-secured.sh.j2", + "site.hbase-env.java64_home": "${JAVA_HOME}", + "site.hbase-env.conf_dir": "${@//site/global/app_conf_dir}", + "site.hbase-env.log_dir": "${@//site/global/app_log_dir}", + "site.hbase-env.pid_dir": "${@//site/global/app_pid_dir}", + "site.hbase-env.master_heapsize": "1024m", + "site.hbase-env.regionserver_heapsize": "1024m", + "site.hbase-env.regionserver_xmn_size": "512m", + "site.hbase-env.hbase_opts": "", + "site.hbase-env.client_jaas_config_file": "${@//site/hbase-env/conf_dir}/hbase_client_jaas.conf", + "site.hbase-env.master_jaas_config_file": "${@//site/hbase-env/conf_dir}/hbase_master_jaas.conf", + "site.hbase-env.regionserver_jaas_config_file": "${@//site/hbase-env/conf_dir}/hbase_regionserver_jaas.conf", + + "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}", + "site.hbase-site.hbase.cluster.distributed": "true", + "site.hbase-site.hbase.superuser": "${USER_NAME}", + "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp", + "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local", + "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}", + "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}", + "site.hbase-site.hbase.regionserver.info.port": "0", + "site.hbase-site.hbase.master.info.port": "0", + "site.hbase-site.hbase.regionserver.port": "0", + "site.hbase-site.hbase.master.port": "0", + "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4", + "site.hbase-site.hbase.regionserver.handler.count": "60", + "site.hbase-site.hbase.hregion.majorcompaction": "86400000", + "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38", + "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2", + "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728", + "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true", + "site.hbase-site.hbase.hregion.max.filesize": "10737418240", + "site.hbase-site.hbase.client.scanner.caching": "100", + "site.hbase-site.zookeeper.session.timeout": "30000", + "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760", + "site.hbase-site.hbase.hstore.compactionThreshold": "3", + "site.hbase-site.hbase.hstore.flush.retries.number": "120", + "site.hbase-site.hbase.hstore.blockingStoreFiles": "10", + "site.hbase-site.hfile.block.cache.size": "0.40", + "site.hbase-site.hbase.zookeeper.property.clientPort": "2181", + "site.hbase-site.hbase.zookeeper.useMulti": "true", + "site.hbase-site.hbase.defaults.for.version.skip": "true", + "site.hbase-site.dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket", + + "site.hbase-site.hbase.security.authentication": "kerberos", + "site.core-site.hadoop.security.authentication": "kerberos", + "site.hbase-site.hbase.security.authorization": "true", + "site.hbase-site.hbase.security.access.early_out": "true", + "site.hbase-site.hbase.coprocessor.master.classes": "org.apache.hadoop.hbase.security.access.AccessController", + "site.hbase-site.hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint", + "site.hbase-site.hbase.regionserver.kerberos.principal": "${USER_NAME}/${THIS_HOST}@EXAMPLE.COM", + "site.hbase-site.hbase.regionserver.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab", + "site.hbase-site.hbase.master.kerberos.principal": "${USER_NAME}/${THIS_HOST}@EXAMPLE.COM", + "site.hbase-site.hbase.master.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab", + "site.hbase-site.hbase.rest.kerberos.principal": "${USER_NAME}/${THIS_HOST}@EXAMPLE.COM", + "site.hbase-site.hbase.rest.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab", + "site.hbase-site.hbase.thrift.kerberos.principal": "${USER_NAME}/${THIS_HOST}@EXAMPLE.COM", + "site.hbase-site.hbase.thrift.keytab.file": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.HBASE.service.keytab", + + "site.hbase-metrics2.template.file": "hadoop-metrics2-hbase.properties-RS.j2", + "site.hbase-metrics2.app_name": "${CLUSTER_NAME}", + "site.hbase-metrics2.metric_collector_host": "${NN_HOST}", + "site.hbase-metrics2.metric_collector_port": "6188", + "site.hbase-metrics2.metric_collector_lib": "", + + "site.hbase-jaas.master_keytab_path": "${@//site/hbase-site/hbase.master.keytab.file}", + "site.hbase-jaas.master_jaas_princ": "${@//site/hbase-site/hbase.master.kerberos.principal}", + "site.hbase-jaas.regionserver_keytab_path": "${@//site/hbase-site/hbase.regionserver.keytab.file}", + "site.hbase-jaas.regionserver_jaas_princ": "${@//site/hbase-site/hbase.regionserver.kerberos.principal}" + }, + "components": { + "slider-appmaster": { + "jvm.heapsize": "1024M", + "slider.hdfs.keytab.dir": ".slider/keytabs/hbase", + "slider.am.login.keytab.name": "${USER_NAME}.headless.keytab", + "slider.keytab.principal.name": "${USER_NAME}" + }, + "HBASE_MASTER": { + "site.global.hbase_component": "master", + "site.hbase-metrics2.template.file": "hadoop-metrics2-hbase.properties-MASTER.j2", + "site.hbase-jaas.template.file": "hbase_master_jaas.conf.j2" + }, + "HBASE_REGIONSERVER": { + "site.global.hbase_component": "regionserver", + "site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2" + }, + "HBASE_REST": { + "site.global.hbase_component": "rest", + "site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2" + }, + "HBASE_THRIFT": { + "site.global.hbase_component": "thrift", + "site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2" + }, + "HBASE_THRIFT2": { + "site.global.hbase_component": "thrift2", + "site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2" + }, + "HBASE_CLIENT": { + "site.hbase-jaas.template.file": "hbase_client_jaas.conf.j2" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/clientInstallConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/clientInstallConfig-default.json b/app-packages/hbase-nopkg/clientInstallConfig-default.json new file mode 100644 index 0000000..f195e7f --- /dev/null +++ b/app-packages/hbase-nopkg/clientInstallConfig-default.json @@ -0,0 +1,6 @@ +{ + "schema":"http://example.org/specification/v2.0.0";, + "global":{ + "client_root": "hbase-${hbase.version}" + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/metainfo-secured.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/metainfo-secured.xml b/app-packages/hbase-nopkg/metainfo-secured.xml new file mode 100644 index 0000000..bc5759e --- /dev/null +++ b/app-packages/hbase-nopkg/metainfo-secured.xml @@ -0,0 +1,255 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>HBASE</name> + <comment> + Apache HBase is the Hadoop database, a distributed, scalable, big data store. + Requirements: + 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner. + 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance. + </comment> + <version>${pkg.version}</version> + <type>YARN-APP</type> + <minHadoopVersion>2.1.0</minHadoopVersion> + <exportedConfigs>hbase-site</exportedConfigs> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.jmx</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </export> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </export> + <export> + <name>org.apache.slider.hbase.rest</name> + <value>http://${HBASE_REST_HOST}:${site.global.rest_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift2</name> + <value>http://${HBASE_THRIFT2_HOST}:${site.global.thrift2_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift</name> + <value>http://${HBASE_THRIFT_HOST}:${site.global.thrift_port}</value> + </export> + <export> + <name>org.apache.slider.metrics</name> + <value>http://${site.hbase-metrics2.metric_collector_host}:${site.hbase-metrics2.metric_collector_port}/ws/v1/timeline/metrics</value> + </export> + <export> + <name>org.apache.slider.hbase.zk-path</name> + <value>${site.hbase-site.hbase.zookeeper.quorum}:${site.hbase-site.zookeeper.znode.parent}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + <commandOrders> + <commandOrder> + <command>HBASE_REGIONSERVER-START</command> + <requires>HBASE_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>HBASE_MASTER-START</command> + <requires>HBASE_REST-INSTALLED</requires> + <requires>HBASE_THRIFT-INSTALLED</requires> + <requires>HBASE_THRIFT2-INSTALLED</requires> + </commandOrder> + </commandOrders> + <components> + <component> + <name>HBASE_MASTER</name> + <category>MASTER</category> + <minInstanceCount>1</minInstanceCount> + <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.hbase.zk-path</appExports> + <componentExports> + <componentExport> + <name>org.apache.slider.jmx</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </componentExport> + <componentExport> + <name>org.apache.slider.monitor</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </componentExport> + <componentExport> + <name>org.apache.slider.hbase.zk-path</name> + <value>${site.hbase-site.hbase.zookeeper.quorum}:${site.hbase-site.zookeeper.znode.parent}</value> + </componentExport> + </componentExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start master</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop master</exec> + <name>STOP</name> + </command> + </commands> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_master_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + + <component> + <name>HBASE_REGIONSERVER</name> + <category>SLAVE</category> + <minInstanceCount>1</minInstanceCount> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start regionserver</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop regionserver</exec> + <name>STOP</name> + </command> + </commands> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_regionserver_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + + <component> + <name>HBASE_REST</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start rest -p {$conf:@//site/global/rest_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop rest</exec> + <name>STOP</name> + </command> + </commands> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_rest_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + + <component> + <name>HBASE_THRIFT</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start thrift -p {$conf:@//site/global/thrift_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop thrift</exec> + <name>STOP</name> + </command> + </commands> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_thrift2_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + + <component> + <name>HBASE_THRIFT2</name> + <category>MASTER</category> + <minInstanceCount>0</minInstanceCount> + <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start thrift2 -p {$conf:@//site/global/thrift_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop thrift2</exec> + <name>STOP</name> + </command> + </commands> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_thrift_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + + <component> + <name>HBASE_CLIENT</name> + <category>CLIENT</category> + <configFile> + <type>template</type> + <fileName>app/conf/hbase_client_jaas.conf</fileName> + <dictionaryName>hbase-jaas</dictionaryName> + </configFile> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/hbase-${pkg.version}.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <packages> + <package> + <type>tarball</type> + <name>hbase-${pkg.version}-bin.tar.gz</name> + </package> + </packages> + + <configFiles> + <configFile> + <type>xml</type> + <fileName>app/conf/hbase-site.xml</fileName> + <dictionaryName>hbase-site</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>app/conf/hbase-env.sh</fileName> + <dictionaryName>hbase-env</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>app/conf/log4j.properties</fileName> + <dictionaryName>hbase-log4j</dictionaryName> + </configFile> + <configFile> + <type>xml</type> + <fileName>app/conf/hbase-policy.xml</fileName> + <dictionaryName>hbase-policy</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>app/conf/hadoop-metrics2-hbase.properties</fileName> + <dictionaryName>hbase-metrics2</dictionaryName> + </configFile> + </configFiles> + + </application> +</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/metainfo.xml b/app-packages/hbase-nopkg/metainfo.xml new file mode 100644 index 0000000..60ef10b --- /dev/null +++ b/app-packages/hbase-nopkg/metainfo.xml @@ -0,0 +1,225 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>HBASE</name> + <comment> + Apache HBase is the Hadoop database, a distributed, scalable, big data store. + Requirements: + 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner. + 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance. + </comment> + <version>${pkg.version}</version> + <type>YARN-APP</type> + <minHadoopVersion>2.1.0</minHadoopVersion> + <exportedConfigs>hbase-site</exportedConfigs> + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.jmx</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </export> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </export> + <export> + <name>org.apache.slider.hbase.rest</name> + <value>http://${HBASE_REST_HOST}:${site.global.rest_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift2</name> + <value>http://${HBASE_THRIFT2_HOST}:${site.global.thrift2_port}</value> + </export> + <export> + <name>org.apache.slider.hbase.thrift</name> + <value>http://${HBASE_THRIFT_HOST}:${site.global.thrift_port}</value> + </export> + <export> + <name>org.apache.slider.metrics</name> + <value>http://${site.hbase-metrics2.metric_collector_host}:${site.hbase-metrics2.metric_collector_port}/ws/v1/timeline/metrics</value> + </export> + <export> + <name>org.apache.slider.hbase.zk-path</name> + <value>${site.hbase-site.hbase.zookeeper.quorum}:${site.hbase-site.zookeeper.znode.parent}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + <commandOrders> + <commandOrder> + <command>HBASE_REGIONSERVER-START</command> + <requires>HBASE_MASTER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>HBASE_MASTER-START</command> + <requires>HBASE_REST-INSTALLED</requires> + <requires>HBASE_THRIFT-INSTALLED</requires> + <requires>HBASE_THRIFT2-INSTALLED</requires> + </commandOrder> + </commandOrders> + <components> + <component> + <name>HBASE_MASTER</name> + <category>MASTER</category> + <minInstanceCount>1</minInstanceCount> + <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.hbase.zk-path</appExports> + <componentExports> + <componentExport> + <name>org.apache.slider.jmx</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value> + </componentExport> + <componentExport> + <name>org.apache.slider.monitor</name> + <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value> + </componentExport> + <componentExport> + <name>org.apache.slider.hbase.zk-path</name> + <value>${site.hbase-site.hbase.zookeeper.quorum}:${site.hbase-site.zookeeper.znode.parent}</value> + </componentExport> + </componentExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start master</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop master</exec> + <name>STOP</name> + </command> + </commands> + </component> + + <component> + <name>HBASE_REGIONSERVER</name> + <category>SLAVE</category> + <minInstanceCount>1</minInstanceCount> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start regionserver</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop regionserver</exec> + <name>STOP</name> + </command> + </commands> + </component> + + <component> + <name>HBASE_REST</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start rest -p {$conf:@//site/global/rest_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop rest</exec> + <name>STOP</name> + </command> + </commands> + </component> + + <component> + <name>HBASE_THRIFT</name> + <category>MASTER</category> + <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start thrift -p {$conf:@//site/global/thrift_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop thrift</exec> + <name>STOP</name> + </command> + </commands> + </component> + + <component> + <name>HBASE_THRIFT2</name> + <category>MASTER</category> + <minInstanceCount>0</minInstanceCount> + <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports> + <commands> + <command> + <exec>{$conf:@//site/global/daemon_cmd} start thrift2 -p {$conf:@//site/global/thrift_port}</exec> + </command> + <command> + <exec>{$conf:@//site/global/daemon_cmd} stop thrift2</exec> + <name>STOP</name> + </command> + </commands> + </component> + + <component> + <name>HBASE_CLIENT</name> + <category>CLIENT</category> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/hbase-${pkg.version}.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <packages> + <package> + <type>archive</type> + <name>hbase-${pkg.version}-bin.tar.gz</name> + </package> + </packages> + + <configFiles> + <configFile> + <type>xml</type> + <fileName>hbase-site.xml</fileName> + <dictionaryName>hbase-site</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>hbase-env.sh</fileName> + <dictionaryName>hbase-env</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>log4j.properties</fileName> + <dictionaryName>hbase-log4j</dictionaryName> + </configFile> + <configFile> + <type>xml</type> + <fileName>hbase-policy.xml</fileName> + <dictionaryName>hbase-policy</dictionaryName> + </configFile> + <configFile> + <type>template</type> + <fileName>hadoop-metrics2-hbase.properties</fileName> + <dictionaryName>hbase-metrics2</dictionaryName> + </configFile> + </configFiles> + + </application> +</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/pom.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/pom.xml b/app-packages/hbase-nopkg/pom.xml new file mode 100644 index 0000000..fcba6f2 --- /dev/null +++ b/app-packages/hbase-nopkg/pom.xml @@ -0,0 +1,92 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <parent> + <groupId>org.apache.slider.packages</groupId> + <artifactId>slider-app-packages</artifactId> + <version>0.91.0-incubating-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>slider-hbase-resources</artifactId> + <packaging>pom</packaging> + <name>Slider HBase Resources</name> + <description>Slider HBase Resources</description> + <properties> + <work.dir>package-tmp</work.dir> + <app.package.name>${project.artifactId}-${pkg.version}</app.package.name> + </properties> + + <profiles> + <profile> + <id>hbase-resources</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>${maven-antrun-plugin.version}</version> + <executions> + <execution> + <id>copy</id> + <phase>validate</phase> + <configuration> + <target name="copy and rename file"> + <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven-assembly-plugin.version}</version> + <configuration> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptor>src/assembly/hbase.xml</descriptor> + <appendAssemblyId>false</appendAssemblyId> + <finalName>${app.package.name}</finalName> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + + </plugins> + </build> + </profile> + </profiles> + + <build> + </build> + + <dependencies> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources-default.json b/app-packages/hbase-nopkg/resources-default.json new file mode 100644 index 0000000..de3b279 --- /dev/null +++ b/app-packages/hbase-nopkg/resources-default.json @@ -0,0 +1,47 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.component.instances": "1", + "yarn.vcores": "1" + }, + "components": { + "slider-appmaster": { + "yarn.memory": "1024" + }, + "HBASE_MASTER": { + "yarn.role.priority": "1", + "yarn.component.instances": "1", + "yarn.placement.escalate.seconds": "10", + "yarn.memory": "1500" + }, + "HBASE_REGIONSERVER": { + "yarn.role.priority": "2", + "yarn.component.instances": "1", + "yarn.memory": "1500", + "yarn.container.failure.threshold": "15", + "yarn.placement.escalate.seconds": "60" + }, + "HBASE_REST": { + "yarn.role.priority": "3", + "yarn.component.instances": "1", + "yarn.component.placement.policy": "1", + "yarn.memory": "556" + }, + "HBASE_THRIFT": { + "yarn.role.priority": "4", + "yarn.component.instances": "1", + "yarn.component.placement.policy": "1", + "yarn.memory": "556" + }, + "HBASE_THRIFT2": { + "yarn.role.priority": "5", + "yarn.component.instances": "1", + "yarn.component.placement.policy": "1", + "yarn.memory": "556" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 new file mode 100644 index 0000000..3ed7af2 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +hbase.period=10 +hbase.servers={{metric_collector_host}}:{{metric_collector_port}} + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +jvm.period=10 +jvm.servers={{metric_collector_host}}:{{metric_collector_port}} + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +rpc.period=10 +rpc.servers={{metric_collector_host}}:{{metric_collector_port}} + +#Ganglia following hadoop example +hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +hbase.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 new file mode 100644 index 0000000..3ed7af2 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +hbase.period=10 +hbase.servers={{metric_collector_host}}:{{metric_collector_port}} + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +jvm.period=10 +jvm.servers={{metric_collector_host}}:{{metric_collector_port}} + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +rpc.period=10 +rpc.servers={{metric_collector_host}}:{{metric_collector_port}} + +#Ganglia following hadoop example +hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 +hbase.sink.ganglia.period=10 + +# default for supportsparse is false +*.sink.ganglia.supportsparse=true + +.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +hbase.sink.ganglia.servers={{metric_collector_host}}:{{metric_collector_port}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-MASTER.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-MASTER.j2 b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-MASTER.j2 new file mode 100644 index 0000000..5473ff5 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-MASTER.j2 @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +*.timeline.plugin.urls={{metric_collector_lib}} +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.serviceName={{app_name}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-RS.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-RS.j2 b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-RS.j2 new file mode 100644 index 0000000..5473ff5 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hadoop-metrics2-hbase.properties-RS.j2 @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +*.timeline.plugin.urls={{metric_collector_lib}} +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.serviceName={{app_name}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 new file mode 100644 index 0000000..2511a29 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 @@ -0,0 +1,49 @@ +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false +export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase-env-secured.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase-env-secured.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env-secured.sh.j2 new file mode 100644 index 0000000..9ee901d --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase-env-secured.sh.j2 @@ -0,0 +1,63 @@ +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log" +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false +export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}" + +export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase-env.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase-env.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env.sh.j2 new file mode 100644 index 0000000..3ed354f --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase-env.sh.j2 @@ -0,0 +1,59 @@ +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log" +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false +export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase-log4j.properties.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase-log4j.properties.j2 b/app-packages/hbase-nopkg/resources/hbase-log4j.properties.j2 new file mode 100644 index 0000000..e0a7e22 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase-log4j.properties.j2 @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=DEBUG +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase_client_jaas.conf.j2 b/app-packages/hbase-nopkg/resources/hbase_client_jaas.conf.j2 new file mode 100644 index 0000000..bb4279c --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase_client_jaas.conf.j2 @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase_master_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase_master_jaas.conf.j2 b/app-packages/hbase-nopkg/resources/hbase_master_jaas.conf.j2 new file mode 100644 index 0000000..91ce3ef --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase_master_jaas.conf.j2 @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{master_keytab_path}}" +principal="{{master_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/resources/hbase_regionserver_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase_regionserver_jaas.conf.j2 b/app-packages/hbase-nopkg/resources/hbase_regionserver_jaas.conf.j2 new file mode 100644 index 0000000..2a9b9f3 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase_regionserver_jaas.conf.j2 @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{regionserver_keytab_path}}" +principal="{{regionserver_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/app-packages/hbase-nopkg/src/assembly/hbase.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/src/assembly/hbase.xml b/app-packages/hbase-nopkg/src/assembly/hbase.xml new file mode 100644 index 0000000..0558319 --- /dev/null +++ b/app-packages/hbase-nopkg/src/assembly/hbase.xml @@ -0,0 +1,89 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + <id>hbase_v${hbase.version}</id> + <formats> + <format>zip</format> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <files> + <file> + <source>appConfig-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>appConfig-secured-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>clientInstallConfig-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>metainfo.xml</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>metainfo-secured.xml</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>${pkg.src}/${pkg.name}</source> + <outputDirectory>resources</outputDirectory> + <filtered>false</filtered> + <fileMode>0755</fileMode> + </file> + </files> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <excludes> + <exclude>pom.xml</exclude> + <exclude>src/**</exclude> + <exclude>target/**</exclude> + <exclude>appConfig-default.json</exclude> + <exclude>appConfig-secured-default.json</exclude> + <exclude>clientInstallConfig-default.json</exclude> + <exclude>metainfo.xml</exclude> + <exclude>metainfo-secured.xml</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 52c7459..7c21e07 100644 --- a/pom.xml +++ b/pom.xml @@ -173,6 +173,7 @@ <protobuf.version>2.5.0</protobuf.version> <slf4j.version>1.7.5</slf4j.version> + <snakeyaml.version>1.16</snakeyaml.version> <storm.version>0.9.3</storm.version> <stringtemplate.version>2.4.1</stringtemplate.version> <zookeeper.version>3.4.6</zookeeper.version> @@ -1178,6 +1179,12 @@ <version>${slf4j.version}</version> </dependency> + <dependency> + <groupId>org.yaml</groupId> + <artifactId>snakeyaml</artifactId> + <version>${snakeyaml.version}</version> + </dependency> + <!-- Used for testing --> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-agent/src/main/python/agent/ActionQueue.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py index 7514337..e973337 100644 --- a/slider-agent/src/main/python/agent/ActionQueue.py +++ b/slider-agent/src/main/python/agent/ActionQueue.py @@ -161,7 +161,7 @@ class ActionQueue(threading.Thread): self.commandStatuses.put_command_status(command, in_progress_status, reportResult) store_config = False - if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']: + if 'commandParams' in command and ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']: store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG] store_command = False if 'roleParams' in command and command['roleParams'] is not None and ActionQueue.AUTO_RESTART in command['roleParams']: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py index 561fd6c..df9b6f0 100644 --- a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py +++ b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py @@ -35,16 +35,15 @@ class BasicInstaller(Script): self.configure(env) process_cmd = format("{cmd}") - Execute(process_cmd, - logoutput=False, - wait_for_finish=False, - pid_file=params.pid_file, - poll_after = 5 - ) + Execute(process_cmd) def stop(self, env): import params env.set_params(params) + self.configure(env) + process_cmd = format("{cmd}") + + Execute(process_cmd) def status(self, env): import params http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index eda352f..57507d6 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -254,6 +254,12 @@ </dependency> <dependency> + <groupId>org.yaml</groupId> + <artifactId>snakeyaml</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <scope>compile</scope>
