SLIDER-359. Create hbase package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b3777fde Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b3777fde Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b3777fde Branch: refs/heads/feature/SLIDER-304_hdfs_token_renewal Commit: b3777fde109cf2e9c9cf22d674d571e7ce845918 Parents: 829821a Author: Sumit Mohanty <[email protected]> Authored: Tue Aug 26 18:13:29 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Tue Aug 26 18:13:29 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase-win/README.txt | 1 + app-packages/hbase-win/appConfig.json | 6 +- .../hbase-win/configuration/hbase-env.xml | 67 +--------------- app-packages/hbase-win/metainfo.xml | 10 +++ app-packages/hbase-win/package/scripts/hbase.py | 15 +--- .../hbase-win/package/scripts/hbase_master.py | 8 +- .../package/scripts/hbase_regionserver.py | 8 +- .../hbase-win/package/scripts/hbase_service.py | 53 ++++++------- .../hbase-win/package/scripts/params.py | 82 +++++++------------- app-packages/hbase/appConfig.json | 3 +- app-packages/hbase/package/scripts/hbase.py | 2 + .../core/providers/__init__.py | 1 + .../core/providers/windows/system.py | 29 +++---- .../libraries/functions/check_process_status.py | 44 ++++++++--- 14 files changed, 127 insertions(+), 202 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt index c122d90..625bce6 100644 --- a/app-packages/hbase-win/README.txt +++ b/app-packages/hbase-win/README.txt @@ -22,6 +22,7 @@ Download the tarball for HBase: Copy the hbase tarball to package/files cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files + rm -rf package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME Edit appConfig.json/metainfo.xml and replace all occurances of 0.98.0.2.1.5.0-2047-hadoop2 with the correct value. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json index 52f3aa3..8261f8c 100644 --- a/app-packages/hbase-win/appConfig.json +++ b/app-packages/hbase-win/appConfig.json @@ -21,15 +21,15 @@ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}", "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}", "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}", + "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;", + "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin", "site.hbase-env.hbase_master_heapsize": "1024m", "site.hbase-env.hbase_regionserver_heapsize": "1024m", - "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data", - "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging", + "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}", "site.hbase-site.hbase.superuser": "yarn", "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp", - "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local", "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}", "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}", "site.hbase-site.hbase.regionserver.info.port": "0", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/configuration/hbase-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/configuration/hbase-env.xml b/app-packages/hbase-win/configuration/hbase-env.xml index 554c3c5..fa5686f 100644 --- a/app-packages/hbase-win/configuration/hbase-env.xml +++ b/app-packages/hbase-win/configuration/hbase-env.xml @@ -45,72 +45,9 @@ <!-- hbase-env.sh --> <property> <name>content</name> - <description>This is the jinja template for hbase-env.sh file</description> + <description>This is the jinja template for start command</description> <value> -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -export JAVA_HOME={{java64_home}} - -# HBase Configuration directory -export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}} - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=${HBASE_CLASSPATH} - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log" -export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" -# Uncomment below to enable java garbage collection logging. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" -export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" -# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -export HBASE_LOG_DIR={{log_dir}} - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -export HBASE_PID_DIR={{pid_dir}} - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -export HBASE_MANAGES_ZK=false - -{% if security_enabled %} -export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" -export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" -export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" -{% endif %} + -Xmx{{heap_size}} "-XX:+UseConcMarkSweepGC" "-XX:CMSInitiatingOccupancyFraction=70" "-Djava.net.preferIPv4Stack=true" "-XX:+ForceTimeHighResolution" "-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" -Xloggc:"{{log_dir}}\hbase-{{role_user}}.gc" -Dhbase.log.dir="{{log_dir}}" -Dhbase.log.file="hbase-{{role_user}}.log" -Dhbase.home.dir="{{hbase_root}}" -Dhbase.id.str="{{hbase_instance_name}}" -XX:OnOutOfMemoryError="taskkill /F /PID p" -Dhbase.root.logger="INFO,DRFA" -Djava.library.path="{{java_library_path}}" -Dhbase.security.logger="INFO,DRFAS" -classpath "{{conf_dir}};{{hbase_root}}\lib\*;{{hbase_additional_cp}}" </value> </property> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml index 4ab56ef..315d279 100644 --- a/app-packages/hbase-win/metainfo.xml +++ b/app-packages/hbase-win/metainfo.xml @@ -110,9 +110,19 @@ </configFile> <configFile> <type>env</type> + <fileName>hbase-env.sh</fileName> + <dictionaryName>hbase-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> <fileName>hbase-log4j.properties</fileName> <dictionaryName>hbase-log4j</dictionaryName> </configFile> + <configFile> + <type>env</type> + <fileName>hbase-policy.xml</fileName> + <dictionaryName>hbase-policy</dictionaryName> + </configFile> </configFiles> </application> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase.py b/app-packages/hbase-win/package/scripts/hbase.py index 70601bc..5ae64eb 100644 --- a/app-packages/hbase-win/package/scripts/hbase.py +++ b/app-packages/hbase-win/package/scripts/hbase.py @@ -23,20 +23,10 @@ from resource_management import * import sys import shutil -def hbase(name=None # 'master' or 'regionserver' or 'client' +def hbase(name=None # 'master' or 'regionserver' ): import params - if name in ["master","regionserver"]: - params.HdfsDirectory(params.hbase_hdfs_root_dir, - action="create_delayed" - ) - params.HdfsDirectory(params.hbase_staging_dir, - action="create_delayed", - mode=0711 - ) - params.HdfsDirectory(None, action="create") - Directory( params.conf_dir, owner = params.hbase_user, group = params.user_group, @@ -52,7 +42,6 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' Directory (os.path.join(params.local_dir, "jars"), owner = params.hbase_user, group = params.user_group, - mode=0775, recursive = True ) @@ -66,14 +55,12 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' if (params.log4j_props != None): File(format("{params.conf_dir}/log4j.properties"), - mode=0644, group=params.user_group, owner=params.hbase_user, content=params.log4j_props ) elif (os.path.exists(format("{conf_dir}/log4j.properties"))): File(format("{params.conf_dir}/log4j.properties"), - mode=0644, group=params.user_group, owner=params.hbase_user ) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_master.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_master.py b/app-packages/hbase-win/package/scripts/hbase_master.py index f4e202f..47b2409 100644 --- a/app-packages/hbase-win/package/scripts/hbase_master.py +++ b/app-packages/hbase-win/package/scripts/hbase_master.py @@ -40,9 +40,9 @@ class HbaseMaster(Script): env.set_params(params) self.configure(env) # for security - ##hbase_service( 'master', - ## action = 'start' - ##) + hbase_service( 'master', + action = 'start' + ) def stop(self, env): import params @@ -56,7 +56,7 @@ class HbaseMaster(Script): import status_params env.set_params(status_params) pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid") - ##check_process_status(pid_file) + check_process_status(pid_file) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_regionserver.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_regionserver.py b/app-packages/hbase-win/package/scripts/hbase_regionserver.py index 7114dc8..daa5732 100644 --- a/app-packages/hbase-win/package/scripts/hbase_regionserver.py +++ b/app-packages/hbase-win/package/scripts/hbase_regionserver.py @@ -40,9 +40,9 @@ class HbaseRegionServer(Script): env.set_params(params) self.configure(env) # for security - ##hbase_service( 'regionserver', - ## action = 'start' - ##) + hbase_service( 'regionserver', + action = 'start' + ) def stop(self, env): import params @@ -56,7 +56,7 @@ class HbaseRegionServer(Script): import status_params env.set_params(status_params) pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid") - ##check_process_status(pid_file) + check_process_status(pid_file) if __name__ == "__main__": http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/hbase_service.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py index 96add84..c02526c 100644 --- a/app-packages/hbase-win/package/scripts/hbase_service.py +++ b/app-packages/hbase-win/package/scripts/hbase_service.py @@ -20,32 +20,29 @@ limitations under the License. from resource_management import * + def hbase_service( - name, - action = 'start'): # 'start' or 'stop' or 'status' - - import params - - role = name - cmd = format("{daemon_script} --config {conf_dir}") - pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid") - - daemon_cmd = None - no_op_test = None - - if action == 'start': - daemon_cmd = format("{cmd} start {role}") - if name == 'rest': - daemon_cmd = format("{daemon_cmd} -p {rest_port}") - elif name == 'thrift': - daemon_cmd = format("{daemon_cmd} -p {thrift_port}") - elif name == 'thrift2': - daemon_cmd = format("{daemon_cmd} -p {thrift2_port}") - no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1") - elif action == 'stop': - daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}") - - if daemon_cmd is not None: - Execute ( daemon_cmd, - not_if = no_op_test - ) + name, + action='start'): # 'start' or 'stop' or 'status' + + import params + + pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid") + heap_size = params.master_heapsize + main_class = "org.apache.hadoop.hbase.master.HMaster" + if name == "regionserver": + heap_size = params.regionserver_heapsize + main_class = "org.apache.hadoop.hbase.regionserver.HRegionServer" + + role_user = format("{hbase_user}-{name}") + + rest_of_the_command = InlineTemplate(params.hbase_env_sh_template, [], heap_size=heap_size, role_user=role_user)() + + process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}") + + Execute(process_cmd, + user=params.hbase_user, + logoutput=False, + wait_for_finish=False, + pid_file=pid_file + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase-win/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py index 33e2bbf..421b9f5 100644 --- a/app-packages/hbase-win/package/scripts/params.py +++ b/app-packages/hbase-win/package/scripts/params.py @@ -25,83 +25,57 @@ import status_params # server configurations config = Script.get_config() +java64_home = config['hostLevelParams']['java_home'] hbase_root = config['configurations']['global']['app_root'] -conf_dir = format("{hbase_root}/conf") -daemon_script = format("{hbase_root}/bin/hbase-daemon.sh") - +hbase_instance_name = config['configurations']['global']['hbase_instance_name'] hbase_user = status_params.hbase_user -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') user_group = config['configurations']['global']['user_group'] -# this is "hadoop-metrics.properties" for 1.x stacks -metric_prop_file_name = "hadoop-metrics2-hbase.properties" +pid_dir = status_params.pid_dir +tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] +local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], + config['configurations']['hbase-site']) +conf_dir = format("{hbase_root}/conf") +log_dir = config['configurations']['global']['app_log_dir'] +input_conf_files_dir = config['configurations']['global']['app_input_conf_dir'] -# not supporting 32 bit jdk. -java64_home = config['hostLevelParams']['java_home'] +hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] +rest_port = config['configurations']['global']['hbase_rest_port'] +thrift_port = config['configurations']['global']['hbase_thrift_port'] +thrift2_port = config['configurations']['global']['hbase_thrift2_port'] -log_dir = config['configurations']['global']['app_log_dir'] master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize'] - regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize'] regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) -pid_dir = status_params.pid_dir -tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] -local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site']) -input_conf_files_dir = config['configurations']['global']['app_input_conf_dir'] +hbase_env_sh_template = config['configurations']['hbase-env']['content'] +java_library_path = config['configurations']['global']['java_library_path'] +hbase_additional_cp = config['configurations']['global']['hbase_additional_cp'] -client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf")) master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf")) -regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf")) - -ganglia_server_host = default('/configurations/global/ganglia_server_host', '') -ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663') - -rest_port = config['configurations']['global']['hbase_rest_port'] -thrift_port = config['configurations']['global']['hbase_thrift_port'] -thrift2_port = config['configurations']['global']['hbase_thrift2_port'] +regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', + format("{conf_dir}/hbase_regionserver_jaas.conf")) +master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] +regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') if security_enabled: _hostname_lowercase = config['hostname'].lower() - master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) - regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) + master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST', hostname_lowercase) + regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST', hostname_lowercase) - -master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] -regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] -kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +kinit_path_local = functions.get_kinit_path( + [default("kinit_path_local", None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) if security_enabled: kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") else: kinit_cmd = "" -#log4j.properties +# log4j.properties if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])): log4j_props = config['configurations']['hbase-log4j']['content'] else: - log4j_props = None - -hbase_env_sh_template = config['configurations']['hbase-env']['content'] - -hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir'] -hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir'] -#for create_hdfs_directory -hostname = config["hostname"] -hadoop_conf_dir = "/etc/hadoop/conf" -hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab'] -hdfs_user = config['configurations']['global']['hdfs_user'] -kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local -) + log4j_props = None \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase/appConfig.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json index f065f90..9c37ed9 100644 --- a/app-packages/hbase/appConfig.json +++ b/app-packages/hbase/appConfig.json @@ -25,8 +25,7 @@ "site.hbase-env.hbase_master_heapsize": "1024m", "site.hbase-env.hbase_regionserver_heapsize": "1024m", - "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data", - "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging", + "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}", "site.hbase-site.hbase.superuser": "yarn", "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp", "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/app-packages/hbase/package/scripts/hbase.py ---------------------------------------------------------------------- diff --git a/app-packages/hbase/package/scripts/hbase.py b/app-packages/hbase/package/scripts/hbase.py index a979bc6..35897df 100644 --- a/app-packages/hbase/package/scripts/hbase.py +++ b/app-packages/hbase/package/scripts/hbase.py @@ -27,6 +27,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' ): import params + """ if name in ["master","regionserver"]: params.HdfsDirectory(params.hbase_hdfs_root_dir, action="create_delayed" @@ -36,6 +37,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client' mode=0711 ) params.HdfsDirectory(None, action="create") + """ Directory( params.conf_dir, owner = params.hbase_user, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/core/providers/__init__.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/__init__.py b/slider-agent/src/main/python/resource_management/core/providers/__init__.py index c6f5c03..8ebeb77 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/__init__.py +++ b/slider-agent/src/main/python/resource_management/core/providers/__init__.py @@ -54,6 +54,7 @@ PROVIDERS = dict( Service="resource_management.core.providers.windows.service.ServiceProvider", Execute="resource_management.core.providers.windows.system.ExecuteProvider", File="resource_management.core.providers.windows.system.FileProvider", + Directory="resource_management.core.providers.windows.system.DirectoryProvider", Tarball="resource_management.core.providers.windows.tarball.TarballProvider" ), default=dict( http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/core/providers/windows/system.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py index 3165b5d..9514bdb 100644 --- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py +++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py @@ -29,15 +29,6 @@ import os import subprocess import shutil from resource_management.libraries.script import Script -import win32con -from win32security import * -from win32api import * -from winerror import ERROR_INVALID_HANDLE -from win32profile import CreateEnvironmentBlock -from win32process import GetExitCodeProcess, STARTF_USESTDHANDLES, STARTUPINFO, CreateProcessAsUser -from win32event import WaitForSingleObject, INFINITE -import msvcrt -import tempfile def _merge_env(env1, env2, merge_keys=['PYTHONPATH']): @@ -76,25 +67,28 @@ def _merge_env(env1, env2, merge_keys=['PYTHONPATH']): def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, user=None, pid_file_name=None): # TODO implement user Logger.info("Executing %s" % (command)) - cur_token = OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS) - current_env = CreateEnvironmentBlock(cur_token, False) - current_env = _merge_env(current_env, env) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd=cwd, env=current_env, shell=False) + cwd=cwd, env=env, shell=False) + code = None if not wait_for_finish: + Logger.debug("No need to wait for the process to exit. Will leave the process running ...") + code = 0 if pid_file_name: + Logger.debug("Writing the process id %s to file %s" % (str(proc.pid), pid_file_name)) pidfile = open(pid_file_name, 'w') pidfile.write(str(proc.pid)) pidfile.close() - return None, None + Logger.info("Wrote the process id to file %s" % pid_file_name) + return code, None, None if timeout: q = Queue() t = threading.Timer( timeout, on_timeout, [proc, q] ) t.start() - out, err = proc.communicate() - code = proc.returncode + out, err = proc.communicate() + code = proc.returncode + if logoutput and out: Logger.info(out) if logoutput and err: @@ -110,7 +104,7 @@ def _set_file_acl(file, user, rights): raise Fail("Can not remove rights for path {0} and user {1}".format(file, user)) code, out, err = _call_command(acls_modify_cmd) if code != 0: - raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user)) + raise Fail("Can not set rights {0} for path {1} and user {2}".format(rights, file, user)) else: return @@ -194,6 +188,7 @@ class ExecuteProvider(Provider): raise Fail("Failed to execute " + self.resource.command) break except Fail as ex: + Logger.info("Error raised: %s" % str(ex)) if i == self.resource.tries - 1: # last try raise ex else: http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b3777fde/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py index e8ed9f0..e17b5f7 100644 --- a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py +++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py @@ -25,6 +25,11 @@ from resource_management.core.logger import Logger __all__ = ["check_process_status"] import os +import subprocess +import platform + +IS_WINDOWS = platform.system() == "Windows" + def check_process_status(pid_file): """ @@ -43,15 +48,32 @@ def check_process_status(pid_file): except: Logger.debug("Pid file {0} does not exist".format(pid_file)) raise ComponentIsNotRunning() - try: - # Kill will not actually kill the process - # From the doc: - # If sig is 0, then no signal is sent, but error checking is still - # performed; this can be used to check for the existence of a - # process ID or process group ID. - os.kill(pid, 0) - except OSError: - Logger.debug("Process with pid {0} is not running. Stale pid file" - " at {1}".format(pid, pid_file)) - raise ComponentIsNotRunning() + + if IS_WINDOWS: + not_running = True + try: + ps = subprocess.Popen(r'tasklist.exe /NH /FI "PID eq %d"' % (pid), + shell=True, stdout=subprocess.PIPE) + output = ps.stdout.read() + ps.stdout.close() + ps.wait() + not_running = str(pid) not in output + except OSError, e: + Logger.debug("Error {0}".format(str(e))) + Logger.info("Process with pid {0} is not running. Stale pid file" + " at {1}".format(pid, pid_file)) + if not_running: + raise ComponentIsNotRunning() + else: + try: + # Kill will not actually kill the process + # From the doc: + # If sig is 0, then no signal is sent, but error checking is still + # performed; this can be used to check for the existence of a + # process ID or process group ID. + os.kill(pid, 0) + except OSError: + Logger.info("Process with pid {0} is not running. Stale pid file" + " at {1}".format(pid, pid_file)) + raise ComponentIsNotRunning() pass
