Repository: incubator-slider Updated Branches: refs/heads/develop e23893cdd -> 522983bd9
SLIDER-763. Add install client command to slider Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/522983bd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/522983bd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/522983bd Branch: refs/heads/develop Commit: 522983bd9a760110697ccbd93dfd5c29872a1656 Parents: e23893c Author: Sumit Mohanty <[email protected]> Authored: Thu Feb 19 13:38:12 2015 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Thu Feb 19 13:40:51 2015 -0800 ---------------------------------------------------------------------- .../slider-pkg/clientInstallConfig-default.json | 8 + .../command-logger/slider-pkg/metainfo.xml | 10 + .../slider-pkg/package/scripts/cl_client.py | 48 +++ .../slider-pkg/package/scripts/client_params.py | 29 ++ .../storm/clientInstallConfig-default.json | 6 + app-packages/storm/metainfo.xml | 10 + app-packages/storm/package/files/storm-slider | 63 ++++ .../storm/package/files/storm-slider.py | 370 +++++++++++++++++++ .../storm/package/scripts/client_params.py | 28 ++ .../storm/package/scripts/storm_client.py | 56 +++ .../package/templates/storm-slider-env.sh.j2 | 38 ++ app-packages/storm/src/assembly/storm.xml | 7 + .../org/apache/slider/client/SliderClient.java | 66 +++- .../apache/slider/client/SliderClientAPI.java | 14 +- .../slider/common/params/ActionClientArgs.java | 65 ++++ .../slider/common/params/ActionPackageArgs.java | 4 +- .../apache/slider/common/params/Arguments.java | 3 +- .../apache/slider/common/params/ClientArgs.java | 19 +- .../slider/common/params/SliderActions.java | 3 + .../providers/AbstractClientProvider.java | 20 + .../providers/agent/AgentClientProvider.java | 304 ++++++++++++++- .../providers/agent/AgentProviderService.java | 7 +- .../test_command_log/client-config.json | 7 + .../agent/actions/TestActionPackage.groovy | 20 +- .../agent/TestAgentClientProvider2.java | 173 ++++++++- .../lifecycle/AgentClientInstallIT.groovy | 79 ++++ .../clusters/remote/slider/slider-client.xml | 10 +- 27 files changed, 1434 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json b/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json new file mode 100644 index 0000000..f0b84a5 --- /dev/null +++ b/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json @@ -0,0 +1,8 @@ +{ + "schema":"http://example.org/specification/v2.0.0";, + "global":{ + "client_root":"{app_install_dir}", + "application_id": "application_id_1" + } +} + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml b/app-packages/command-logger/slider-pkg/metainfo.xml index 5de2c37..eb02f6a 100644 --- a/app-packages/command-logger/slider-pkg/metainfo.xml +++ b/app-packages/command-logger/slider-pkg/metainfo.xml @@ -36,6 +36,16 @@ <timeout>600</timeout> </commandScript> </component> + + <component> + <name>COMMAND_LOGGER_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/cl_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> </components> <osSpecifics> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py ---------------------------------------------------------------------- diff --git a/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py b/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py new file mode 100644 index 0000000..73a2675 --- /dev/null +++ b/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +import os +from resource_management import * + + +class Client(Script): + def install(self, env): + import client_params + self.install_packages(env) + File(format(client_params.client_root + '/operations.log'), + mode=0755, + content=Template('operations.log.j2', container_id = client_params.container_id, application_id = client_params.application_id) + ) + + def configure(self, env): + pass + + def start(self, env): + pass + + def stop(self, env): + pass + + def status(self, env): + pass + +if __name__ == "__main__": + Client().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/package/scripts/client_params.py ---------------------------------------------------------------------- diff --git a/app-packages/command-logger/slider-pkg/package/scripts/client_params.py b/app-packages/command-logger/slider-pkg/package/scripts/client_params.py new file mode 100644 index 0000000..4d5a4ff --- /dev/null +++ b/app-packages/command-logger/slider-pkg/package/scripts/client_params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import * + +config = Script.get_config() + +app_install_dir = config['configurations']['global']['app_install_dir'] +client_root = config['configurations']['global']['client_root'] + +container_id = config['configurations']['global']['container_id'] +application_id = config['configurations']['global']['application_id'] + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/clientInstallConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/clientInstallConfig-default.json b/app-packages/storm/clientInstallConfig-default.json new file mode 100644 index 0000000..898439d --- /dev/null +++ b/app-packages/storm/clientInstallConfig-default.json @@ -0,0 +1,6 @@ +{ + "schema":"http://example.org/specification/v2.0.0";, + "global":{ + "client_root":"{app_install_dir}/apache-storm-${pkg.version}" + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml index d2625f4..cc96628 100644 --- a/app-packages/storm/metainfo.xml +++ b/app-packages/storm/metainfo.xml @@ -125,6 +125,16 @@ <timeout>600</timeout> </commandScript> </component> + + <component> + <name>STORM_CLIENT</name> + <category>CLIENT</category> + <commandScript> + <script>scripts/storm_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> </components> <osSpecifics> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/files/storm-slider ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/storm-slider b/app-packages/storm/package/files/storm-slider new file mode 100644 index 0000000..af492bd --- /dev/null +++ b/app-packages/storm/package/files/storm-slider @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Copyright 2014 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Resolve links - $0 may be a softlink +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +# find python >= 2.6 +if [ -a /usr/bin/python2.6 ]; then + PYTHON=/usr/bin/python2.6 +fi + +if [ -z "$PYTHON" ]; then + PYTHON=/usr/bin/python +fi + +# check for version +majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1` +minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2` +numversion=$(( 10 * $majversion + $minversion)) +if (( $numversion < 26 )); then + echo "Need python version > 2.6" + exit 1 +fi + +STORM_BIN_DIR=`dirname ${PRG}` +STORM_BASE_DIR=`cd ${STORM_BIN_DIR}/..;pwd` +export STORM_BASE_DIR + +export STORM_CONF_DIR="${STORM_CONF_DIR:-$STORM_BASE_DIR/conf}" + +if [ -f "${STORM_CONF_DIR}/storm-slider-env.sh" ]; then + . "${STORM_CONF_DIR}/storm-slider-env.sh" +fi + +$PYTHON ${STORM_BIN_DIR}/storm-slider.py $@ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/files/storm-slider.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/files/storm-slider.py b/app-packages/storm/package/files/storm-slider.py new file mode 100644 index 0000000..cbe441d --- /dev/null +++ b/app-packages/storm/package/files/storm-slider.py @@ -0,0 +1,370 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from os.path import expanduser +import sys, tempfile +import json +import subprocess as sub +import re +import shlex + +def is_windows(): + return sys.platform.startswith('win') + +def identity(x): + return x + +def cygpath(x): + command = ["cygpath", "-wp", x] + p = sub.Popen(command,stdout=sub.PIPE) + output, errors = p.communicate() + lines = output.split("\n") + return lines[0] + +if sys.platform == "cygwin": + normclasspath = cygpath +else: + normclasspath = identity + +SLIDER_DIR = os.getenv('SLIDER_HOME', None) + +if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)): + print "Unable to find SLIDER_HOME. Please configure SLIDER_HOME before running storm-slider" + sys.exit(1) + +USER_CONF_DIR = os.path.expanduser("~/.storm") + +if os.getenv('STORM_BASE_DIR', None) != None: + STORM_DIR = os.getenv('STORM_BASE_DIR', None) +elif os.getenv('STORM_HOME', None) != None: + STORM_DIR = os.getenv('STORM_HOME', None) +else: + print "Either STORM_BASE_DIR or STORM_HOME must be set." + sys.exit(1) + +CMD_OPTS = {} +CONFIG_OPTS = [] +JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', '')) +pid = os.getpid() +CONFFILE = os.path.join(tempfile.gettempdir(),"storm."+str(pid)+".json") +SLIDER_CLIENT_CONF = os.path.join(SLIDER_DIR,'conf','slider-client.xml') +SLIDER_CMD = os.path.join(SLIDER_DIR,'bin','slider.py') +JAVA_HOME = os.getenv('JAVA_HOME', None) +JAVA_CMD= 'java' if not JAVA_HOME else os.path.join(JAVA_HOME, 'bin', 'java') +if is_windows(): + JAVA_CMD = JAVA_CMD + '.exe' +STORM_THRIFT_TRANSPORT_KEY = "storm.thrift.transport" +NIMBUS_HOST_KEY = "nimbus.host" +NIMBUS_THRIFT_PORT_KEY = "nimbus.thrift.port" + +if not os.path.isfile(JAVA_CMD): + print "Unable to find "+JAVA_CMD+" please check JAVA_HOME" + sys.exit(1) + +def get_config_opts(): + global CONFIG_OPTS + return "-Dstorm.options=" + (','.join(CONFIG_OPTS)).replace(' ', "%%%%") + +def get_jars_full(adir): + files = os.listdir(adir) + ret = [] + for f in files: + if f.endswith(".jar"): + ret.append(os.path.join(adir, f)) + return ret + +def get_classpath(extrajars): + ret = (get_jars_full(os.path.join(STORM_DIR ,"lib"))) + ret.extend(extrajars) + + sep = ";" if is_windows() else ":" + return normclasspath(sep.join(ret)) + +def print_remoteconfvalue(name): + """Syntax: [storm-slider --app remoteconfvalue conf-name] + + Prints out the value for conf-name in the cluster's Storm configs. + This command must be run on a cluster machine. + """ + storm_conf = storm_conf_values([name]) + for conf in storm_conf: + print conf + +def parse_args(string): + r"""Takes a string of whitespace-separated tokens and parses it into a list. + Whitespace inside tokens may be quoted with single quotes, double quotes or + backslash (similar to command-line arguments in bash). + + >>> parse_args(r'''"a a" 'b b' c\ c "d'd" 'e"e' 'f\'f' "g\"g" "i""i" 'j''j' k" "k l' l' mm n\\n''') + ['a a', 'b b', 'c c', "d'd", 'e"e', "f'f", 'g"g', 'ii', 'jj', 'k k', 'l l', 'mm', r'n\n'] + """ + re_split = re.compile(r'''((?: + [^\s"'\\] | + "(?: [^"\\] | \\.)*" | + '(?: [^'\\] | \\.)*' | + \\. + )+)''', re.VERBOSE) + args = re_split.split(string)[1::2] + args = [re.compile(r'"((?:[^"\\]|\\.)*)"').sub('\\1', x) for x in args] + args = [re.compile(r"'((?:[^'\\]|\\.)*)'").sub('\\1', x) for x in args] + return [re.compile(r'\\(.)').sub('\\1', x) for x in args] + +def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False): + global CONFFILE + storm_log_dir = os.path.join(STORM_DIR, "logs") + all_args = [ + "java", jvmtype, get_config_opts(), + "-Dstorm.home=" + STORM_DIR, + "-cp", get_classpath(extrajars), + ] + jvmopts + [klass] + list(args) + print "Running: " + " ".join(all_args) + if is_windows(): + sub.call([JAVA_CMD] + all_args[1:]) + else: + os.execvp(JAVA_CMD, all_args) # replaces the current process and never returns + +def jar(jarfile, klass, *args): + """Syntax: [storm-slider --app jar topology-jar-path class ...] + + Runs the main method of class with the specified arguments. + The storm jars and configs in ~/.storm are put on the classpath. + The process is configured so that StormSubmitter + (http://nathanmarz.github.com/storm/doc/backtype/storm/StormSubmitter.html) + will upload the jar at topology-jar-path when the topology is submitted. + """ + exec_storm_class( + klass, + jvmtype="-client", + extrajars=[jarfile, USER_CONF_DIR, os.path.join(STORM_DIR ,"bin")], + args=args, + jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile]) + +def kill(*args): + """Syntax: [storm-slider --app kill topology-name [-w wait-time-secs]] + + Kills the topology with the name topology-name. Storm will + first deactivate the topology's spouts for the duration of + the topology's message timeout to allow all messages currently + being processed to finish processing. Storm will then shutdown + the workers and clean up their state. You can override the length + of time Storm waits between deactivation and shutdown with the -w flag. + """ + exec_storm_class( + "backtype.storm.command.kill_topology", + args=args, + jvmtype="-client", + extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")]) + +def activate(*args): + """Syntax: [storm-slider --app activate topology-name] + + Activates the specified topology's spouts. + """ + exec_storm_class( + "backtype.storm.command.activate", + args=args, + jvmtype="-client", + extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")]) + +def listtopos(*args): + """Syntax: [storm-slider --app list] + + List the running topologies and their statuses. + """ + exec_storm_class( + "backtype.storm.command.list", + args=args, + jvmtype="-client", + extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")]) + +def deactivate(*args): + """Syntax: [storm-slider --app deactivate topology-name] + + Deactivates the specified topology's spouts. + """ + exec_storm_class( + "backtype.storm.command.deactivate", + args=args, + jvmtype="-client", + extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")]) + +def rebalance(*args): + """Syntax: [storm-slider --app rebalance topology-name [-w wait-time-secs] [-n new-num-workers] [-e component=parallelism]*] + + Sometimes you may wish to spread out where the workers for a topology + are running. For example, let's say you have a 10 node cluster running + 4 workers per node, and then let's say you add another 10 nodes to + the cluster. You may wish to have Storm spread out the workers for the + running topology so that each node runs 2 workers. One way to do this + is to kill the topology and resubmit it, but Storm provides a "rebalance" + command that provides an easier way to do this. + + Rebalance will first deactivate the topology for the duration of the + message timeout (overridable with the -w flag) and then redistribute + the workers evenly around the cluster. The topology will then return to + its previous state of activation (so a deactivated topology will still + be deactivated and an activated topology will go back to being activated). + + The rebalance command can also be used to change the parallelism of a running topology. + Use the -n and -e switches to change the number of workers or number of executors of a component + respectively. + """ + exec_storm_class( + "backtype.storm.command.rebalance", + args=args, + jvmtype="-client", + extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR,"bin")]) + +def version(): + """Syntax: [storm-slider --app version] + Prints the version number of this Storm release. + """ + releasefile = os.path.join(STORM_DIR, "RELEASE") + if os.path.exists(releasefile): + print open(releasefile).readline().strip() + else: + print "Unknown" + +def quicklinks(): + """Syntax: [storm-slider --app version] + Prints the quicklinks information of storm-slider registry" + """ + global CMD_OPTS + all_args = ["slider", "registry", "--getconf", "quicklinks","--format", "json", "--name"] + if 'app_name' in CMD_OPTS.keys(): + all_args.append(CMD_OPTS['app_name']) + else: + print_usage() + sys.exit(1) + + if 'user' in CMD_OPTS.keys(): + all_args.append( "--user "+CMD_OPTS['user']) + + #os.spawnvp(os.P_WAIT,SLIDER_CMD, all_args) + cmd = [SLIDER_CMD] + all_args[1:] + if is_windows(): + cmd = ['python'] + cmd + + sub.call(cmd) + +def get_storm_config_json(): + global CMD_OPTS + all_args = ["slider.py", "registry", "--getconf", "storm-site","--format", "json", "--dest", CONFFILE, "--name"] + if 'app_name' in CMD_OPTS.keys(): + all_args.append(CMD_OPTS['app_name']) + else: + print_usage() + sys.exit(1) + + if 'user' in CMD_OPTS.keys(): + all_args.append( "--user "+CMD_OPTS['user']) + + #os.spawnvp(os.P_WAIT,SLIDER_CMD, all_args) + cmd = [SLIDER_CMD] + all_args[1:] + + if is_windows(): + cmd = ['python'] + cmd + + sub.call(cmd) + if not os.path.exists(CONFFILE): + print "Failed to read slider deployed storm config" + sys.exit(1) + +def storm_conf_values(keys): + file = open(CONFFILE,"r") + data = json.load(file) + storm_args = [] + for key in keys: + if data.has_key(key): + storm_args.append(key+"="+data[key]) + return storm_args + +def print_commands(): + """Print all client commands and link to documentation""" + print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys())) + print "\nHelp:", "\n\thelp", "\n\thelp <command>" + +def print_usage(command=None): + """Print one help message or list of available commands""" + if command != None: + if COMMANDS.has_key(command): + print (COMMANDS[command].__doc__ or + "No documentation provided for <%s>" % command) + else: + print "<%s> is not a valid command" % command + else: + print "Please provide yarn app name followed by command" + print "storm-slider --app --user" + print_commands() + + +def unknown_command(*args): + print "Unknown command: [storm-slider %s]" % ' '.join(sys.argv[1:]) + print_usage() + +COMMANDS = {"jar": jar, "kill": kill, "remoteconfvalue": print_remoteconfvalue, + "activate": activate, "deactivate": deactivate, "rebalance": rebalance, "help": print_usage, + "list": listtopos, "version": version, "quicklinks" : quicklinks} + +def parse_config(config_list): + global CONFIG_OPTS + if len(config_list) > 0: + for config in config_list: + CONFIG_OPTS.append(config) + +def parse_config_opts(args): + curr = args[:] + curr.reverse() + global CMD_OPTS + global CONFIG_OPTS + args_list = [] + while len(curr) > 0: + token = curr.pop() + if token == "--app": + CMD_OPTS['app_name'] = curr.pop() if (len(curr) != 0) else None + elif token == "--user": + CMD_OPTS['user'] = curr.pop() if (len(curr) != 0) else None + elif token == "-c" and len(curr) != 0: + CONFIG_OPTS.append(curr.pop()) + else: + args_list.append(token) + return args_list + +def main(): + args = parse_config_opts(sys.argv[1:]) + if len(args) < 1: + print_usage() + sys.exit(-1) + COMMAND = args[0] + ARGS = args[1:] + if (COMMAND != 'help'): + get_storm_config_json() + storm_conf = storm_conf_values([NIMBUS_HOST_KEY,NIMBUS_THRIFT_PORT_KEY,STORM_THRIFT_TRANSPORT_KEY]) + parse_config(storm_conf) + try: + (COMMANDS.get(COMMAND, unknown_command))(*ARGS) + except: + print "Unexpected error:", sys.exc_info()[0] + raise + finally: + if (os.path.isfile(CONFFILE)): + os.remove(CONFFILE) + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/scripts/client_params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/client_params.py b/app-packages/storm/package/scripts/client_params.py new file mode 100644 index 0000000..924a202 --- /dev/null +++ b/app-packages/storm/package/scripts/client_params.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import * + +config = Script.get_config() + +app_install_dir = config['configurations']['global']['app_install_dir'] +client_root = config['configurations']['global']['client_root'] + +slider_home_dir = config['configurations']['global']['slider_home_dir'] +java64_home = config['configurations']['global']['java64_home'] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/scripts/storm_client.py ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/scripts/storm_client.py b/app-packages/storm/package/scripts/storm_client.py new file mode 100644 index 0000000..6d36edd --- /dev/null +++ b/app-packages/storm/package/scripts/storm_client.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from storm import storm +from service import service + +class Client(Script): + def install(self, env): + import client_params + self.install_packages(env) + File(format(client_params.client_root + '/bin/storm-slider'), + content=StaticFile("storm-slider"), + mode=0755 + ) + File(format(client_params.client_root + '/bin/storm-slider.py'), + content=StaticFile("storm-slider.py"), + mode=0755 + ) + File(format(client_params.client_root + '/conf/storm-slider-env.sh'), + mode=0755, + content=Template('storm-slider-env.sh.j2', java64_home = client_params.java64_home, slider_home_dir = client_params.slider_home_dir) + ) + + def configure(self, env): + pass + + def start(self, env): + pass + + def stop(self, env): + pass + + def status(self, env): + pass + +if __name__ == "__main__": + Client().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/templates/storm-slider-env.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm/package/templates/storm-slider-env.sh.j2 b/app-packages/storm/package/templates/storm-slider-env.sh.j2 new file mode 100644 index 0000000..8022a4b --- /dev/null +++ b/app-packages/storm/package/templates/storm-slider-env.sh.j2 @@ -0,0 +1,38 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} +#!/usr/bin/env python +# -*- coding: utf-8 -*- +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +export JAVA_HOME={{java64_home}} +export SLIDER_HOME={{slider_home_dir}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/src/assembly/storm.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml index f7dcf13..6a6e750 100644 --- a/app-packages/storm/src/assembly/storm.xml +++ b/app-packages/storm/src/assembly/storm.xml @@ -42,6 +42,12 @@ <fileMode>0755</fileMode> </file> <file> + <source>clientInstall-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> <source>metainfo.xml</source> <outputDirectory>/</outputDirectory> <filtered>true</filtered> @@ -65,6 +71,7 @@ <exclude>target/**</exclude> <exclude>appConfig-default.json</exclude> <exclude>appConfig-secured-default.json</exclude> + <exclude>clientInstall-default.json</exclude> <exclude>metainfo.xml</exclude> </excludes> <fileMode>0755</fileMode> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 5cbe945..77b5e96 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -78,6 +78,7 @@ import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionExistsArgs; import org.apache.slider.common.params.ActionInstallKeytabArgs; @@ -172,6 +173,8 @@ import java.io.StringWriter; import java.io.Writer; import java.net.InetSocketAddress; import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -204,7 +207,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private String deployedClusterName; /** - * Cluster opaerations against the deployed cluster -will be null + * Cluster operations against the deployed cluster -will be null * if no bonding has yet taken place */ private SliderClusterOperations sliderClusterOperations; @@ -385,6 +388,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // actions if (ACTION_PACKAGE.equals(action)) { exitCode = actionPackage(serviceArgs.getActionPackageArgs()); + } else if (ACTION_CLIENT.equals(action)) { + exitCode = actionClient(serviceArgs.getActionClientArgs()); } else if (ACTION_INSTALL_PACKAGE.equals(action)) { exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); } else if (ACTION_INSTALL_KEYTAB.equals(action)) { @@ -869,6 +874,65 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override + public int actionClient(ActionClientArgs clientInfo) throws + SliderException, + IOException { + + if(!clientInfo.install) { + throw new BadCommandArgumentsException( + "Only install command is supported for the client.\n" + + CommonArgs.usage(serviceArgs, ACTION_CLIENT)); + } + + if (clientInfo.installLocation == null) { + throw new BadCommandArgumentsException( + "A valid install location must be provided for the client.\n" + + CommonArgs.usage(serviceArgs, ACTION_CLIENT)); + } else { + if (!clientInfo.installLocation.exists()) { + throw new BadCommandArgumentsException("Install path does not exist at " + + clientInfo.installLocation.getAbsolutePath()); + } + if (!clientInfo.installLocation.isDirectory()) { + throw new BadCommandArgumentsException("Install path is not a valid directory " + + clientInfo.installLocation.getAbsolutePath()); + } + } + + File pkgFile; + if (StringUtils.isEmpty(clientInfo.packageURI)) { + throw new BadCommandArgumentsException("A valid application package location required."); + } else { + pkgFile = new File(clientInfo.packageURI); + if (!pkgFile.exists() || pkgFile.isDirectory()) { + throw new BadCommandArgumentsException("Unable to access supplied pkg file at " + + pkgFile.getAbsolutePath()); + } + } + + JSONObject config = null; + if(clientInfo.clientConfig != null) { + try { + byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath()); + config = new JSONObject(new String(encoded, Charset.defaultCharset())); + }catch(JSONException jsonEx) { + log.error("Unable to read supplied config", jsonEx); + throw new SliderException("Invalid configuration. Must be a valid json file.", jsonEx); + } + } + + // Only INSTALL is supported + AbstractClientProvider provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE); + provider.processClientOperation(sliderFileSystem, + "INSTALL", + clientInfo.installLocation, + pkgFile, + config); + return EXIT_SUCCESS; + } + + + @Override public int actionPackage(ActionPackageArgs actionPackageInfo) throws YarnException, IOException { if (actionPackageInfo.install) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 4fd4136..efb1f7f 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; +import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionFlexArgs; @@ -125,6 +126,17 @@ public interface SliderClientAPI extends Service { throws YarnException, IOException; /** + * Perform client operations such as install or configure + * + * @param clientInfo the arguments needed for client operations + * + * @throws SliderException bad arguments. + * @throws IOException problems related to package and destination folders + */ + int actionClient(ActionClientArgs clientInfo) + throws IOException, SliderException; + + /** * Managing slider application package * * @param pkgInfo the arguments needed to upload, delete or list the package @@ -283,7 +295,7 @@ public interface SliderClientAPI extends Service { /** * diagnostic operation * - * @param diagosticArgs diagnostic Arguments + * @param diagnosticArgs diagnostic Arguments * @return 0 for success, -1 for some issues that aren't errors, just * failures to retrieve information (e.g. no application name * specified) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java new file mode 100644 index 0000000..4154c9f --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.File; + +@Parameters(commandNames = {SliderActions.ACTION_CLIENT}, + commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT) + +public class ActionClientArgs extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_CLIENT; + } + + @Parameter(names = {ARG_INSTALL}, + description = "Install client") + public boolean install; + + @Parameter(names = {ARG_PACKAGE}, + description = "Path to app package") + public String packageURI; + + @Parameter(names = {ARG_DEST}, + description = "The location where to install the client") + public File installLocation; + + @Parameter(names = {ARG_CONFIG}, + description = "Client configuration") + public File clientConfig; + + /** + * Get the min #of params expected + * + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public int getMaxParams() { + return 1; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java index d5bd293..e98aba1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java @@ -22,7 +22,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @Parameters(commandNames = {SliderActions.ACTION_PACKAGE}, - commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE) + commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE) public class ActionPackageArgs extends AbstractActionArgs { @@ -31,7 +31,7 @@ public class ActionPackageArgs extends AbstractActionArgs { return SliderActions.ACTION_PACKAGE; } - @Parameter(names = {ARG_PKGINSTALL}, + @Parameter(names = {ARG_INSTALL}, description = "Install package operation") public boolean install; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 511d01e..789b285 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -81,7 +81,7 @@ public interface Arguments { String ARG_PACKAGE = "--package"; String ARG_PATH = "--path"; String ARG_PKGDELETE = "--delete"; - String ARG_PKGINSTALL = "--install"; + String ARG_INSTALL = "--install"; String ARG_PKGINSTANCES = "--instances"; String ARG_PKGLIST = "--list"; String ARG_PROVIDER = "--provider"; @@ -105,6 +105,7 @@ public interface Arguments { String ARG_ZKHOSTS = "--zkhosts"; String ARG_ZKPATH = "--zkpath"; String ARG_ZKPORT = "--zkport"; + String ARG_CONFIG = "--config"; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index fc466a1..fcc1b81 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -58,6 +58,7 @@ public class ClientArgs extends CommonArgs { private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs(); private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs(); private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs(); + private final ActionClientArgs actionClientArgs = new ActionClientArgs(); private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs(); private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs(); private final ActionKillContainerArgs actionKillContainerArgs = @@ -96,6 +97,7 @@ public class ClientArgs extends CommonArgs { actionHelpArgs, actionInstallPackageArgs, actionPackageArgs, + actionClientArgs, actionInstallKeytabArgs, actionKeytabArgs, actionKillContainerArgs, @@ -141,17 +143,15 @@ public class ClientArgs extends CommonArgs { return actionBuildArgs; } - public ActionInstallPackageArgs getActionInstallPackageArgs() { - return actionInstallPackageArgs; } + public ActionInstallPackageArgs getActionInstallPackageArgs() { return actionInstallPackageArgs; } - public ActionPackageArgs getActionPackageArgs() { - return actionPackageArgs; } + public ActionClientArgs getActionClientArgs() { return actionClientArgs; } - public ActionInstallKeytabArgs getActionInstallKeytabArgs() { - return actionInstallKeytabArgs; } + public ActionPackageArgs getActionPackageArgs() { return actionPackageArgs; } - public ActionKeytabArgs getActionKeytabArgs() { - return actionKeytabArgs; } + public ActionInstallKeytabArgs getActionInstallKeytabArgs() { return actionInstallKeytabArgs; } + + public ActionKeytabArgs getActionKeytabArgs() { return actionKeytabArgs; } public ActionUpdateArgs getActionUpdateArgs() { return actionUpdateArgs; @@ -257,6 +257,9 @@ public class ClientArgs extends CommonArgs { } else if (SliderActions.ACTION_PACKAGE.equals(action)) { bindCoreAction(actionPackageArgs); + } else if (SliderActions.ACTION_CLIENT.equals(action)) { + bindCoreAction(actionClientArgs); + } else if (SliderActions.ACTION_INSTALL_KEYTAB.equals(action)) { bindCoreAction(actionInstallKeytabArgs); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index e58ad2d..2599927 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -48,6 +48,7 @@ public interface SliderActions { String ACTION_INSTALL_PACKAGE = "install-package"; String ACTION_PACKAGE = "package"; String ACTION_INSTALL_KEYTAB = "install-keytab"; + String ACTION_CLIENT = "client"; String ACTION_KEYTAB = "keytab"; String DESCRIBE_ACTION_AM_SUICIDE = "Tell the Slider Application Master to simulate a process failure by terminating itself"; @@ -86,6 +87,8 @@ public interface SliderActions { String DESCRIBE_ACTION_VERSION = "Print the Slider version information"; String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages"; + String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use this package"; + String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory"; String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab file in the sub-folder 'keytabs' of the user's Slider base directory"; String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index f89f842..b5aafd4 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -28,9 +28,11 @@ import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.AbstractLauncher; +import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.List; @@ -218,4 +220,22 @@ public abstract class AbstractClientProvider extends Configured { return Collections.emptySet(); } + /** + * Process client operations for applications such as install, configure + * @param fileSystem + * @param operation + * @param config + * @param clientPackage + * @param clientInstallPath + * @throws SliderException + */ + public void processClientOperation(SliderFileSystem fileSystem, + String operation, + File clientInstallPath, + File clientPackage, + JSONObject config) + throws SliderException { + throw new SliderException("Provider does not support client operations."); + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 0ef8a33..94d4c97 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -18,7 +18,12 @@ package org.apache.slider.providers.agent; +import com.google.common.io.Files; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.slider.api.InternalKeys; @@ -38,16 +43,34 @@ import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.providers.agent.application.metadata.MetainfoParser; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; /** This class implements the client-side aspects of the agent deployer */ public class AgentClientProvider extends AbstractClientProvider @@ -229,7 +252,7 @@ public class AgentClientProvider extends AbstractClientProvider Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT); log.info("Automatically uploading the agent tarball at {}", agentPath); fileSystem.getFileSystem().mkdirs(agentPath); - if(ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { + if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { instanceDefinition.getInternalOperations().set( InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, new Path(agentPath, AGENT_TAR).toUri()); @@ -249,7 +272,7 @@ public class AgentClientProvider extends AbstractClientProvider throw new SliderException("Error retrieving metainfo", e); } - if(metainfo == null) { + if (metainfo == null) { log.error("Error retrieving metainfo from {}", appDef); throw new SliderException("Error parsing metainfo file, possibly bad structure."); } @@ -262,4 +285,281 @@ public class AgentClientProvider extends AbstractClientProvider return tags; } + + @Override + public void processClientOperation(SliderFileSystem fileSystem, + String operation, + File clientInstallPath, + File appPackage, + JSONObject config) throws SliderException { + // create temp folder + // create sub-folders app_pkg, agent_pkg, command + File tmpDir = Files.createTempDir(); + log.info("Command is being executed at {}", tmpDir.getAbsolutePath()); + File appPkgDir = new File(tmpDir, "app_pkg"); + appPkgDir.mkdir(); + + File agentPkgDir = new File(tmpDir, "agent_pkg"); + agentPkgDir.mkdir(); + + File cmdDir = new File(tmpDir, "command"); + cmdDir.mkdir(); + + Metainfo metaInfo = null; + JSONObject defaultConfig = null; + try { + // expand app package into /app_pkg + ZipInputStream zipInputStream = null; + try { + zipInputStream = new ZipInputStream(new FileInputStream(appPackage)); + { + ZipEntry zipEntry = zipInputStream.getNextEntry(); + while (zipEntry != null) { + if ("metainfo.xml".equals(zipEntry.getName())) { + int size = (int) zipEntry.getSize(); + if (size != -1) { + log.info("Reading {} of size {}", zipEntry.getName(), + zipEntry.getSize()); + byte[] content = new byte[size]; + int offset = 0; + while (offset < size) { + offset += zipInputStream.read(content, offset, size - offset); + } + metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content)); + } + } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { + int size = (int) zipEntry.getSize(); + if (size != -1) { + log.info("Reading {} of size {}", zipEntry.getName(), + zipEntry.getSize()); + byte[] content = new byte[size]; + int offset = 0; + while (offset < size) { + offset += zipInputStream.read(content, offset, size - offset); + } + try { + defaultConfig = new JSONObject(new String(content, Charset.defaultCharset())); + } catch (JSONException jex) { + throw new SliderException("Unable to read default client config.", jex); + } + } + } + String filePath = appPkgDir + File.separator + zipEntry.getName(); + if (!zipEntry.isDirectory()) { + extractFile(zipInputStream, filePath); + } else { + File dir = new File(filePath); + dir.mkdir(); + } + zipInputStream.closeEntry(); + zipEntry = zipInputStream.getNextEntry(); + } + } + } finally { + zipInputStream.close(); + } + + if (metaInfo == null) { + throw new SliderException("Not a valid app package. Could not read metainfo.xml."); + } + + expandAgentTar(agentPkgDir); + + JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath); + FileWriter file = new FileWriter(new File(cmdDir, "command.json")); + try { + file.write(commandJson.toString()); + + } catch (IOException e) { + e.printStackTrace(); + } finally { + file.flush(); + file.close(); + } + + String client_script = null; + for (Component component : metaInfo.getApplication().getComponents()) { + if (component.getCategory().equals("CLIENT")) { + client_script = component.getCommandScript().getScript(); + log.info("Installing CLIENT {} using script {}", component.getName(), client_script); + break; + } + } + + if (SliderUtils.isUnset(client_script)) { + throw new SliderException("No valid CLIENT component found. Aborting install."); + } + + runCommand(appPkgDir, agentPkgDir, cmdDir, client_script); + + } catch (IOException ioex) { + log.warn("Error while executing INSTALL command {}", ioex.getMessage()); + throw new SliderException("INSTALL client failed."); + } + } + + protected void runCommand( + File appPkgDir, + File agentPkgDir, + File cmdDir, + String clientScript) throws SliderException { + int exitCode = 0; + Exception exp = null; + try { + String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" + + File.separator + clientScript; + List<String> command = Arrays.asList(AgentKeys.PYTHON_EXE, + "-S", + clientScriptPath, + "INSTALL", + cmdDir.getAbsolutePath() + File.separator + "command.json", + appPkgDir.getAbsolutePath() + File.separator + "package", + cmdDir.getAbsolutePath() + File.separator + "command-out.json", + "DEBUG"); + ProcessBuilder pb = new ProcessBuilder(command); + log.info("Command: " + StringUtils.join(pb.command(), " ")); + pb.environment().put(SliderKeys.PYTHONPATH, + agentPkgDir.getAbsolutePath() + + File.separator + "slider-agent:" + + agentPkgDir.getAbsolutePath() + + File.separator + "slider-agent/jinja2"); + log.info("{}={}", SliderKeys.PYTHONPATH, pb.environment().get(SliderKeys.PYTHONPATH)); + + Process proc = pb.start(); + InputStream stderr = proc.getErrorStream(); + InputStream stdout = proc.getInputStream(); + BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout)); + BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr)); + + proc.waitFor(); + + String line; + while ((line = stdOutReader.readLine()) != null) { + log.info("Stdout: " + line); + } + while ((line = stdErrReader.readLine()) != null) { + log.info("Stderr: " + line); + } + + exitCode = proc.exitValue(); + log.info("Exit value is {}", exitCode); + } catch (IOException e) { + exp = e; + } catch (InterruptedException e) { + exp = e; + } + + if (exitCode != 0) { + throw new SliderException("INSTALL client failed with exit code " + exitCode); + } + + if (exp != null) { + log.error("Error while executing INSTALL command {}. Stack trace {}", + exp.getMessage(), + ExceptionUtils.getStackTrace(exp)); + throw new SliderException("INSTALL client failed.", exp); + } + } + + private void expandAgentTar(File agentPkgDir) throws IOException { + String libDirProp = + System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR); + TarArchiveInputStream tarIn = new TarArchiveInputStream( + new GzipCompressorInputStream( + new BufferedInputStream( + new FileInputStream(tarFile) + ) + ) + ); + try { + TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); + while (tarEntry != null) { + File destPath = new File(agentPkgDir, tarEntry.getName()); + if (tarEntry.isDirectory()) { + destPath.mkdirs(); + } else { + destPath.createNewFile(); + byte[] byteToRead = new byte[1024]; + BufferedOutputStream buffOut = + new BufferedOutputStream(new FileOutputStream(destPath)); + try { + int len; + while ((len = tarIn.read(byteToRead)) != -1) { + buffOut.write(byteToRead, 0, len); + } + } finally { + buffOut.close(); + } + } + tarEntry = tarIn.getNextTarEntry(); + } + } finally { + tarIn.close(); + } + } + + protected JSONObject getCommandJson(JSONObject defaultConfig, + JSONObject inputConfig, + Metainfo metainfo, + File clientInstallPath) throws SliderException { + try { + JSONObject pkgList = new JSONObject(); + pkgList.put(AgentKeys.PACKAGE_LIST, + AgentProviderService.getPackageListFromApplication(metainfo.getApplication())); + JSONObject obj = new JSONObject(); + obj.put("hostLevelParams", pkgList); + + JSONObject configuration = new JSONObject(); + JSONObject global = new JSONObject(); + global.put("app_install_dir", clientInstallPath.getAbsolutePath()); + + if (defaultConfig != null) { + readConfigEntries(defaultConfig, clientInstallPath, global); + } + if (inputConfig != null) { + readConfigEntries(inputConfig, clientInstallPath, global); + } + + configuration.put("global", global); + obj.put("configurations", configuration); + return obj; + } catch (JSONException jex) { + log.warn("Error while executing INSTALL command {}", jex.getMessage()); + throw new SliderException("INSTALL client failed."); + } + } + + private void readConfigEntries(JSONObject inpConfig, + File clientInstallPath, + JSONObject globalConfig) + throws JSONException { + JSONObject globalSection = inpConfig.getJSONObject("global"); + Iterator it = globalSection.keys(); + while (it.hasNext()) { + String key = (String) it.next(); + String value = globalSection.getString(key); + if (SliderUtils.isSet(value)) { + value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath()); + } + if (globalConfig.has(key)) { + // last one wins + globalConfig.remove(key); + } + globalConfig.put(key, value); + } + } + + private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException { + BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath)); + try { + byte[] bytesRead = new byte[4096]; + int read = 0; + while ((read = zipInputStream.read(bytesRead)) != -1) { + output.write(bytesRead, 0, read); + } + } finally { + output.close(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 0730beb..60236a5 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -1663,11 +1663,10 @@ public class AgentProviderService extends AbstractProviderService implements response.addExecutionCommand(cmd); } - private String getPackageList() { + protected static String getPackageListFromApplication(Application application) { String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}"; String pkgListFormatString = "[%s]"; List<String> packages = new ArrayList(); - Application application = getMetainfo().getApplication(); if (application != null) { List<OSSpecific> osSpecifics = application.getOSSpecifics(); if (osSpecifics != null && osSpecifics.size() > 0) { @@ -1688,6 +1687,10 @@ public class AgentProviderService extends AbstractProviderService implements } } + private String getPackageList() { + return getPackageListFromApplication(getMetainfo().getApplication()); + } + private void prepareExecutionCommand(ExecutionCommand cmd) { cmd.setTaskId(taskId.incrementAndGet()); cmd.setCommandId(cmd.getTaskId() + "-1"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/app_packages/test_command_log/client-config.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_command_log/client-config.json b/slider-core/src/test/app_packages/test_command_log/client-config.json new file mode 100644 index 0000000..c4cdac6 --- /dev/null +++ b/slider-core/src/test/app_packages/test_command_log/client-config.json @@ -0,0 +1,7 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "global": { + "java_home": "/usr/jdk64/jdk1.7.0_67", + "container_id": "cid_1" + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy index b0b832a..fa6145e 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy @@ -52,7 +52,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL + Arguments.ARG_INSTALL ], ) fail("expected an exception, got a status code " + launcher.serviceExitCode) @@ -70,7 +70,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase" ], ) @@ -89,7 +89,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", ], @@ -100,7 +100,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", ], @@ -120,7 +120,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "unlikely_to_be_a_file_path", ], @@ -140,7 +140,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", ], @@ -151,7 +151,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "hbase", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", Arguments.ARG_REPLACE_PKG @@ -171,7 +171,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_PKGLIST ], ) @@ -189,7 +189,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_PKGINSTANCES ], ) @@ -207,7 +207,7 @@ class TestActionPackage extends AgentMiniClusterTestBase { //varargs list of command line params [ SliderActions.ACTION_PACKAGE, - Arguments.ARG_PKGINSTALL, + Arguments.ARG_INSTALL, Arguments.ARG_NAME, "storm", Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", ], http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java index 0ff613a..1e4d834 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java @@ -16,15 +16,27 @@ */ package org.apache.slider.providers.agent; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.log4j.BasicConfigurator; import org.apache.slider.api.InternalKeys; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.providers.ProviderUtils; +import org.apache.slider.providers.agent.application.metadata.Application; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.providers.agent.application.metadata.OSPackage; +import org.apache.slider.providers.agent.application.metadata.OSSpecific; +import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -35,6 +47,12 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; @@ -42,13 +60,19 @@ import static org.easymock.EasyMock.expect; * */ @RunWith(PowerMockRunner.class) -@PrepareForTest(ProviderUtils.class) +@PrepareForTest({ProviderUtils.class, ProcessBuilder.class, AgentClientProvider.class}) public class TestAgentClientProvider2 { protected static final Logger log = LoggerFactory.getLogger(TestAgentClientProvider2.class); @Rule public TemporaryFolder folder = new TemporaryFolder(); + @BeforeClass + public static void initialize() { + BasicConfigurator.resetConfiguration(); + BasicConfigurator.configure(); + } + @Test public void testPrepareAMAndConfigForLaunch() throws Exception { AgentClientProvider provider = new AgentClientProvider(null); @@ -85,4 +109,151 @@ public class TestAgentClientProvider2 { PowerMock.verify(sfs, fs, ProviderUtils.class); Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH)); } + + @Test + public void testGetCommandJson() throws Exception { + AgentClientProvider provider = new AgentClientProvider(null); + JSONObject defaultConfig = null; + + JSONObject inputConfig = new JSONObject(); + JSONObject global = new JSONObject(); + global.put("a", "b"); + global.put("d", "{app_install_dir}/d"); + inputConfig.put("global", global); + + Metainfo metainfo = new Metainfo(); + Application app = new Application(); + metainfo.setApplication(app); + OSSpecific osSpecific = new OSSpecific(); + osSpecific.setOsType("any"); + app.addOSSpecific(osSpecific); + OSPackage pkg = new OSPackage(); + osSpecific.addOSPackage(pkg); + pkg.setName("app.tar"); + pkg.setType("tarball"); + + File clientInstallPath = new File("/tmp/file1"); + + JSONObject output = provider.getCommandJson(defaultConfig, + inputConfig, + metainfo, + clientInstallPath); + JSONObject outConfigs = output.getJSONObject("configurations"); + Assert.assertNotNull(outConfigs); + JSONObject outGlobal = outConfigs.getJSONObject("global"); + Assert.assertNotNull(outGlobal); + Assert.assertEquals("b", outGlobal.getString("a")); + Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d")); + + defaultConfig = new JSONObject(); + global = new JSONObject(); + global.put("a1", "b2"); + global.put("a", "b-not"); + global.put("d1", "{app_install_dir}/d"); + defaultConfig.put("global", global); + + output = provider.getCommandJson(defaultConfig, + inputConfig, + metainfo, + clientInstallPath); + outConfigs = output.getJSONObject("configurations"); + Assert.assertNotNull(outConfigs); + outGlobal = outConfigs.getJSONObject("global"); + Assert.assertNotNull(outGlobal); + Assert.assertEquals("b", outGlobal.getString("a")); + Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d")); + Assert.assertEquals("b2", outGlobal.getString("a1")); + Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d1")); + } + + + @Test + public void testRunCommand() throws Exception { + AgentClientProvider provider = new AgentClientProvider(null); + File appPkgDir = new File("/tmp/pkg"); + File agentPkgDir = new File("/tmp/agt"); + File cmdDir = new File("/tmp/cmd"); + String client_script = "scripts/abc.py"; + + List<String> commands = + Arrays.asList("python", "-S", "/tmp/pkg/package/scripts/abc.py", "INSTALL", "/tmp/cmd/command.json", + "/tmp/pkg/package", "/tmp/cmd/command-out.json", "DEBUG"); + ProcessBuilder pbMock = PowerMock.createMock(ProcessBuilder.class); + Process procMock = PowerMock.createMock(Process.class); + PowerMock.expectNew(ProcessBuilder.class, commands).andReturn(pbMock); + + expect(pbMock.environment()).andReturn(new HashMap<String, String>()).anyTimes(); + expect(pbMock.start()).andReturn(procMock); + expect(pbMock.command()).andReturn(new ArrayList<String>()); + expect(procMock.waitFor()).andReturn(0); + expect(procMock.exitValue()).andReturn(0); + expect(procMock.getErrorStream()).andReturn(IOUtils.toInputStream("stderr", "UTF-8")); + expect(procMock.getInputStream()).andReturn(IOUtils.toInputStream("stdout", "UTF-8")); + + PowerMock.replayAll(); + + provider.runCommand(appPkgDir, + agentPkgDir, + cmdDir, + client_script); + PowerMock.verifyAll(); + } + + @Test + public void testSliderClientForInstallFailures() throws Exception { + SliderClient client = new SliderClient(); + client.bindArgs(new Configuration(), "client", "--dest", "a_random_path/none", "--package", "a_random_pkg.zip"); + ActionClientArgs args = new ActionClientArgs(); + args.install = false; + try { + client.actionClient(args); + }catch(BadCommandArgumentsException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Only install command is supported for the client")); + } + + args.install = true; + try { + client.actionClient(args); + }catch(BadCommandArgumentsException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("A valid install location must be provided for the client")); + } + + File tmpFile = File.createTempFile("del", ""); + File dest = new File(tmpFile.getParentFile(), tmpFile.getName() + "dir"); + args.installLocation = dest; + try { + client.actionClient(args); + }catch(BadCommandArgumentsException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Install path does not exist at")); + } + + dest.mkdir(); + try { + client.actionClient(args); + }catch(BadCommandArgumentsException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("A valid application package location required")); + } + + tmpFile = File.createTempFile("del", ".zip"); + args.packageURI = tmpFile.toString(); + args.clientConfig = tmpFile; + try { + client.actionClient(args); + }catch(SliderException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Invalid configuration. Must be a valid json file")); + } + + args.clientConfig = null; + try { + client.actionClient(args); + }catch(SliderException e) { + log.info(e.getMessage()); + Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo.xml")); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy new file mode 100644 index 0000000..ef75821 --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.lifecycle + +import groovy.io.FileType +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.api.records.YarnApplicationState +import org.apache.slider.api.ClusterDescription +import org.apache.slider.api.StatusKeys +import org.apache.slider.client.SliderClient +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderXmlConfKeys +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.funtest.framework.AgentCommandTestBase +import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.funtest.framework.SliderShell +import org.junit.After +import org.junit.Before +import org.junit.Test + +@CompileStatic +@Slf4j +public class AgentClientInstallIT extends AgentCommandTestBase + implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + + + @Test + public void testAgentClientInstall() throws Throwable { + + describe "Install command logger client" + File zipFileName = new File(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE).canonicalFile + File tmpFile = File.createTempFile("del", ""); + File dest = new File(tmpFile.getParentFile(), tmpFile.getName() + "dir"); + String CLIENT_CONFIG = + "../slider-core/src/test/app_packages/test_command_log/client-config.json" + + dest.mkdir() + + SliderShell shell = slider(EXIT_SUCCESS, + [ + ACTION_CLIENT, + "--install", + ARG_DEST, dest.canonicalPath, + ARG_PACKAGE, zipFileName.absolutePath, + ARG_CONFIG, CLIENT_CONFIG + ]) + logShell(shell) + + def list = [] + + dest.eachFileRecurse (FileType.FILES) { file -> + list << file.toString() + } + + String expectedFile1 = new File(dest, "operations.log").toString(); + String expectedFile2 = new File(new File(dest, "command-logger-app"), "operations.log").toString(); + + assert list.contains(expectedFile1) + assert list.contains(expectedFile2) + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/src/test/clusters/remote/slider/slider-client.xml ---------------------------------------------------------------------- diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml index f8d88eb..41dc32e 100644 --- a/src/test/clusters/remote/slider/slider-client.xml +++ b/src/test/clusters/remote/slider/slider-client.xml @@ -31,22 +31,22 @@ <property> <name>slider.zookeeper.quorum</name> - <value>c6403.ambari.apache.org:2181</value> + <value>c6401.ambari.apache.org:2181</value> </property> <property> <name>yarn.resourcemanager.address</name> - <value>c6403.ambari.apache.org:8050</value> + <value>c6401.ambari.apache.org:8050</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> - <value>c6403.ambari.apache.org:8030</value> + <value>c6401.ambari.apache.org:8030</value> </property> <property> <name>fs.defaultFS</name> - <value>hdfs://c6403.ambari.apache.org:8020</value> + <value>hdfs://c6401.ambari.apache.org:8020</value> </property> <property> @@ -74,7 +74,7 @@ <property> <name>slider.test.agent.tar</name> - <value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value> + <value>hdfs://c6401.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value> </property> </configuration>
