Package simplification poc-I
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a31d20ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a31d20ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a31d20ee Branch: refs/heads/feature/packaging_improvements Commit: a31d20eedd323209d7d6116b67aa7b2eecf3b39a Parents: 8d0e337 Author: Sumit Mohanty <[email protected]> Authored: Tue Feb 24 20:17:00 2015 -0800 Committer: Sumit Mohanty <[email protected]> Committed: Tue Feb 24 20:17:00 2015 -0800 ---------------------------------------------------------------------- pom.xml | 2 +- .../python/agent/CustomServiceOrchestrator.py | 87 +++++- .../src/main/python/scripts/basic_installer.py | 55 ++++ slider-agent/src/main/python/scripts/params.py | 29 ++ .../org/apache/slider/api/InternalKeys.java | 8 + .../org/apache/slider/client/SliderClient.java | 147 +++++++++- .../org/apache/slider/common/SliderKeys.java | 12 +- .../AbstractClusterBuildingActionArgs.java | 31 +- .../slider/common/params/AddonArgsDelegate.java | 54 ++++ .../apache/slider/common/params/Arguments.java | 3 + .../slider/common/tools/CoreFileSystem.java | 27 +- .../apache/slider/common/tools/SliderUtils.java | 56 +++- .../slider/core/build/InstanceBuilder.java | 6 + .../slider/core/persist/InstancePaths.java | 4 + .../providers/agent/AgentClientProvider.java | 29 +- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 284 ++++++++++++------- .../slider/providers/agent/AgentUtils.java | 38 ++- .../providers/agent/ComponentCommandOrder.java | 6 +- .../agent/application/metadata/Metainfo.java | 12 + .../application/metadata/json/Application.java | 123 ++++++++ .../application/metadata/json/CommandOrder.java | 57 ++++ .../metadata/json/CommandScript.java | 66 +++++ .../application/metadata/json/Component.java | 129 +++++++++ .../metadata/json/ComponentCommand.java | 79 ++++++ .../application/metadata/json/ConfigFile.java | 53 ++++ .../agent/application/metadata/json/Export.java | 57 ++++ .../application/metadata/json/ExportGroup.java | 57 ++++ .../application/metadata/json/MetaInfo.java | 177 ++++++++++++ .../metadata/json/MetaInfoParser.java | 85 ++++++ .../application/metadata/json/Package.java | 54 ++++ .../common/tools/TestWindowsSupport.groovy | 2 +- .../agent/TestAgentClientProvider2.java | 16 +- .../agent/TestAgentProviderService.java | 131 +++------ .../agent/TestComponentCommandOrder.java | 2 +- .../metadata/json/TestMetaInfoParser.java | 115 ++++++++ 36 files changed, 1830 insertions(+), 265 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 75e60bd..6c5f2d5 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ <!-- Java versions --> - <project.java.src.version>1.6</project.java.src.version> + <project.java.src.version>1.7</project.java.src.version> <enforced.java.version>${project.java.src.version}</enforced.java.version> <groovy.version>2.2.2</groovy.version> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py index 119c926..51ce975 100644 --- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py +++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py @@ -76,23 +76,36 @@ class CustomServiceOrchestrator(): override_output_files=True, store_command=False): allocated_ports = {} try: + py_file_list = [] + json_path = None + script_type = command['commandParams']['script_type'] - script = command['commandParams']['script'] - timeout = int(command['commandParams']['command_timeout']) task_id = command['taskId'] command_name = command['roleCommand'] - script_path = self.resolve_script_path(self.base_dir, script, script_type) - script_tuple = (script_path, self.base_dir) - tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir, "structured-out-{0}.json".format(task_id))) - if script_type.upper() != self.SCRIPT_TYPE_PYTHON: - # We don't support anything else yet + if script_type.upper() == self.SCRIPT_TYPE_PYTHON: + script = command['commandParams']['script'] + timeout = int(command['commandParams']['command_timeout']) + script_path = self.resolve_script_path(self.base_dir, script, script_type) + script_tuple = (script_path, self.base_dir) + py_file_list = [script_tuple] + + json_path = self.dump_command_to_json(command, allocated_ports, store_command) + elif script_type.upper() == "SHELL": + timeout = int(command['commandParams']['command_timeout']) + + json_path = self.dump_command_to_json2(command, allocated_ports) + script_path = os.path.realpath(posixpath.join(self.config.getWorkRootPath(), + "infra", "agent", "slider-agent", "scripts", "basic_installer.py")) + script_tuple = (script_path, self.base_dir) + py_file_list = [script_tuple] + else: + # We don't support anything else yet message = "Unknown script type {0}".format(script_type) raise AgentException(message) - json_path = self.dump_command_to_json(command, allocated_ports, store_command) - py_file_list = [script_tuple] + # filter None values filtered_py_file_list = [i for i in py_file_list if i] logger_level = logging.getLevelName(logger.level) @@ -212,6 +225,41 @@ class CustomServiceOrchestrator(): return res pass + def dump_command_to_json2(self, command, allocated_ports): + """ + Converts command to json file and returns file path + """ + # Perform few modifications to stay compatible with the way in which + # site.pp files are generated by manifestGenerator.py + command['public_hostname'] = self.public_fqdn + if 'hostname' in command: + command['appmaster_hostname'] = command['hostname'] + command['hostname'] = self.public_fqdn + + # Now, dump the json file + command_type = command['commandType'] + from ActionQueue import ActionQueue # To avoid cyclic dependency + + if command_type == ActionQueue.STATUS_COMMAND: + # These files are frequently created, thats why we don't + # store them all, but only the latest one + file_path = os.path.realpath(posixpath.join(self.tmp_dir, "status_command.json")) + else: + task_id = command['taskId'] + file_path = os.path.realpath(posixpath.join(self.tmp_dir, "command-{0}.json".format(task_id))) + # Json may contain passwords, that's why we need proper permissions + if os.path.isfile(file_path) and os.path.exists(file_path): + os.unlink(file_path) + + self.finalize_command(command, False, allocated_ports) + self.finalize_exec_command(command) + + with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT, + 0644), 'w') as f: + content = json.dumps(command, sort_keys=False, indent=4) + f.write(content) + return file_path + def dump_command_to_json(self, command, allocated_ports, store_command=False): """ Converts command to json file and returns file path @@ -291,6 +339,27 @@ class CustomServiceOrchestrator(): pass """ + configurations/global/exec_cmd should be resolved based on the rest of the config + {$conf:@//site/global/xmx_val} ==> command['configurations']['global']['xmx_val'] + """ + def finalize_exec_command(self, command): + variable_format = "{{$conf:@//site/{0}/{1}}}" + if 'configurations' in command: + exec_cmd = command['configurations']['global']['exec_cmd'] + for key in command['configurations']: + if len(command['configurations'][key]) > 0: + for k, value in command['configurations'][key].items(): + replaced_key = variable_format.format(key, k) + exec_cmd = exec_cmd.replace(replaced_key, value) + pass + pass + pass + command['configurations']['global']['exec_cmd'] = exec_cmd + pass + pass + + + """ All unallocated ports should be set to 0 Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}" or "${component.ALLOCATED_PORT}{DEFAULT_port}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-agent/src/main/python/scripts/basic_installer.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/basic_installer.py b/slider-agent/src/main/python/scripts/basic_installer.py new file mode 100644 index 0000000..561fd6c --- /dev/null +++ b/slider-agent/src/main/python/scripts/basic_installer.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +class BasicInstaller(Script): + def install(self, env): + self.install_packages(env) + pass + + def configure(self, env): + pass + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + process_cmd = format("{cmd}") + + Execute(process_cmd, + logoutput=False, + wait_for_finish=False, + pid_file=params.pid_file, + poll_after = 5 + ) + + def stop(self, env): + import params + env.set_params(params) + + def status(self, env): + import params + env.set_params(params) + check_process_status(params.pid_file) + +if __name__ == "__main__": + BasicInstaller().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-agent/src/main/python/scripts/params.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/params.py b/slider-agent/src/main/python/scripts/params.py new file mode 100644 index 0000000..9051f43 --- /dev/null +++ b/slider-agent/src/main/python/scripts/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +# server configurations +config = Script.get_config() + +app_root = config['configurations']['global']['app_root'] +pid_file = config['configurations']['global']['pid_file'] +exec_cmd = config['configurations']['global']['exec_cmd'] +cmd = format(exec_cmd) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index b360fbe..fd17ed1 100644 --- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -69,6 +69,14 @@ public interface InternalKeys { */ String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path"; /** + * where the app def is stored + */ + String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path"; + /** + * where addons for the app are stored + */ + String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path"; + /** * Time in milliseconds to wait after forking any in-AM * process before attempting to start up the containers: {@value} * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 77b5e96..3ef8c68 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -20,11 +20,13 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.io.Files; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathNotFoundException; @@ -174,7 +176,6 @@ import java.io.Writer; import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -913,7 +914,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe JSONObject config = null; if(clientInfo.clientConfig != null) { try { - byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath()); + byte[] encoded = Files.toByteArray(clientInfo.clientConfig); config = new JSONObject(new String(encoded, Charset.defaultCharset())); }catch(JSONException jsonEx) { log.error("Unable to read supplied config", jsonEx); @@ -1163,6 +1164,89 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } + List<AppDefinition> appDefinitions = new ArrayList<>(); + // if metainfo is provided add to the app instance + if(buildInfo.appMetaInfo != null) { + if(!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) { + throw new BadConfigException("--metainfo file cannot be read."); + } + + if(buildInfo.appDef != null) { + throw new BadConfigException("both --metainfo and --appdef may not be specified."); + } + + if(SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { + throw new BadConfigException("application.def must not be set if --metainfo is provided."); + } + + File tempDir = Files.createTempDir(); + File pkgSrcDir = new File(tempDir, "default"); + pkgSrcDir.mkdirs(); + Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json")); + + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); + log.info("Using default app def path {}", appDirPath.toString()); + AppDefinition ad = new AppDefinition(); + ad.fileInFs = appDirPath; + ad.appDef = pkgSrcDir; + ad.pkgName = SliderKeys.DEFAULT_APP_PKG; + appDefinitions.add(ad); +// processDefinitionPackageOrFolder( +// appDirPath, +// pkgSrcDir, +// SliderKeys.DEFAULT_APP_PKG); + Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); + log.info("Setting app package to {}.", appDefPath); + } + + if (buildInfo.appDef != null) { + if(SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { + throw new BadConfigException("application.def must not be set if --appdef is provided."); + } + + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); + AppDefinition ad = new AppDefinition(); + ad.fileInFs = appDirPath; + ad.appDef = buildInfo.appDef; + ad.pkgName = SliderKeys.DEFAULT_APP_PKG; + appDefinitions.add(ad); +// processDefinitionPackageOrFolder( +// appDirPath, +// buildInfo.appDef, +// SliderKeys.DEFAULT_APP_PKG); + Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); + log.info("Setting app package to {}.", appDefPath); } + + if (buildInfo.addonDelegate.getAddonMap().size() > 0) { + List<String> addons = new ArrayList<String>(); + Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap(); + for (String key : addonMap.keySet()) { + Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); + String addonPkgName = "addon_" + key + ".zip"; + AppDefinition ad = new AppDefinition(); + ad.fileInFs = addonPath; + ad.appDef = buildInfo.appDef; + ad.pkgName = addonPkgName; + appDefinitions.add(ad); +// processDefinitionPackageOrFolder(addonPath, +// buildInfo.appDef, +// addonPkgName); + String addOnKey = AgentKeys.ADDON_PREFIX + key; + Path addonPkgPath = new Path(addonPath, addonPkgName); + log.info("Setting addon package {} to {}.",addOnKey, addonPkgPath); + appConf.getGlobalOptions().set(addOnKey, addonPkgPath); + } + + String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); + if(SliderUtils.isUnset(existingList)) { + existingList = ""; + } + appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ",")); + } + + //get the command line options ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree(); ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree(); @@ -1254,17 +1338,61 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // make any substitutions needed at this stage replaceTokens(appConf.getConfTree(), getUsername(), clustername); - // providers to validate what there is - AggregateConf instanceDescription = builder.getInstanceDescription(); - validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); - validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); + // TODO: Refactor the validation code and persistence code try { persistInstanceDefinition(overwrite, appconfdir, builder); + for(AppDefinition ad : appDefinitions) { + processDefinitionPackageOrFolder(ad.fileInFs, + ad.appDef, + ad.pkgName); + } } catch (LockAcquireFailedException e) { log.warn("Failed to get a Lock on {} : {}", builder, e); throw new BadClusterStateException("Failed to save " + clustername + ": " + e); } + + // providers to validate what there is + AggregateConf instanceDescription = builder.getInstanceDescription(); + validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); + validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); + } + + /** + * Process the application package or folder by copying it to the cluster path + * @param fileInFs fileInFs folder + * @param appDef definition package or folder + * @param pkgName package name when its a folder + * @throws BadConfigException + * @throws IOException + * @throws BadClusterStateException + */ + private void processDefinitionPackageOrFolder(Path fileInFs, + File appDef, + String pkgName) + throws BadConfigException, IOException, BadClusterStateException { + if (!appDef.canRead()) { + throw new BadConfigException("Pkg/Folder cannot be accessed - " + appDef.getAbsolutePath()); + } + + File src = appDef; + String targetName = appDef.getName(); + + if (appDef.isDirectory()) { + log.info("Processing app package/folder {} for {}", appDef.getAbsolutePath(), pkgName); + File tmpDir = Files.createTempDir(); + File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + pkgName); + SliderUtils.zipFolder(appDef, zipFile); + + src = zipFile; + targetName = pkgName; + } + + sliderFileSystem.getFileSystem().copyFromLocalFile( + false, + false, + new Path(src.toURI()), + new Path(fileInFs, targetName)); } protected void persistInstanceDefinition(boolean overwrite, @@ -3728,6 +3856,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } return EXIT_SUCCESS; } + + // TODO: Refactor this + class AppDefinition { + Path fileInFs; + File appDef; + String pkgName; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 5cf7022..30ff258 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -168,11 +168,21 @@ public interface SliderKeys extends SliderXmlConfKeys { String TMP_LOGDIR_PREFIX = "/tmp/slider-"; String TMP_DIR_PREFIX = "tmp"; String AM_DIR_PREFIX = "appmaster"; - + + /** + * Store the default app definition, e.g. metainfo file or content of a folder + */ + String APP_DEF_DIR = "appdef"; + /** + * Store additional app defs - co-processors + */ + String ADDONS_DIR = "addons"; + String SLIDER_JAR = "slider.jar"; String JCOMMANDER_JAR = "jcommander.jar"; String GSON_JAR = "gson.jar"; String AGENT_TAR = "slider-agent.tar.gz"; + String DEFAULT_APP_PKG = "appPkg.zip"; String DEFAULT_JVM_HEAP = "256M"; int DEFAULT_YARN_MEMORY = 256; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 56e01c3..f2e3c61 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -74,13 +74,6 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Provider of the specific cluster application") public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE; - /* - - @Parameter(names = {ARG_PACKAGE}, - description = "URI to a slider package", - converter = URIArgumentConverter.class ) - public URI packageURI; - */ @Parameter(names = {ARG_PACKAGE}, description = "URI to a slider package") public String packageURI; @@ -93,6 +86,14 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Template application configuration") public File template; + @Parameter(names = {ARG_METAINFO}, + description = "Application meta info") + public File appMetaInfo; + + @Parameter(names = {ARG_APPDEF}, + description = "Application def (folder or a zip package)") + public File appDef; + @Parameter(names = {ARG_QUEUE}, description = "Queue to submit the application") public String queue; @@ -100,6 +101,9 @@ public abstract class AbstractClusterBuildingActionArgs extends @ParametersDelegate public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + @ParametersDelegate + public AddonArgsDelegate addonDelegate = new AddonArgsDelegate(); + @ParametersDelegate public AppAndResouceOptionArgsDelegate optionsDelegate = @@ -152,6 +156,19 @@ public abstract class AbstractClusterBuildingActionArgs extends return componentDelegate.getComponentMap(); } + @VisibleForTesting + public List<String> getAddonTuples() { + return addonDelegate.getAddonTuples(); + } + + /** + * Get the list of addons (may be empty, but never null) + */ + public Map<String, String> getAddonMap() throws + BadCommandArgumentsException { + return addonDelegate.getAddonMap(); + } + public Path getConfdir() { return confdir; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java new file mode 100644 index 0000000..65ebc4b --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class AddonArgsDelegate extends AbstractArgsDelegate { + + /** + * This is a listing of addon packages + */ + @Parameter(names = {ARG_ADDON}, + arity = 2, + description = "--addon <name> <folder or package>", + splitter = DontSplitArguments.class) + public List<String> addonTuples = new ArrayList<String>(0); + + + /** + * Get the list of addons (may be empty, but never null) + * + * @return map of named addons + * + * @throws BadCommandArgumentsException parse problem + */ + public Map<String, String> getAddonMap() throws BadCommandArgumentsException { + return convertTupleListToMap("addon", addonTuples); + } + + public List<String> getAddonTuples() { + return addonTuples; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 789b285..6fbc240 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -106,6 +106,9 @@ public interface Arguments { String ARG_ZKPATH = "--zkpath"; String ARG_ZKPORT = "--zkport"; String ARG_CONFIG = "--config"; + String ARG_METAINFO = "--metainfo"; + String ARG_ADDON = "--addon"; + String ARG_APPDEF = "--appdef"; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index e835312..3df73df 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -122,6 +122,31 @@ public class CoreFileSystem { } /** + * Build up the path string for app def folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAppDefDirPath(String clustername) { + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.APP_DEF_DIR); + } + + /** + * Build up the path string for addon folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAddonDirPath(String clustername, String addonId) { + Preconditions.checkNotNull(addonId); + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId); + } + + /** * Build up the path string for package install location -no attempt to * create the directory is made * @@ -568,7 +593,7 @@ public class CoreFileSystem { /** * List all application instances persisted for this user, giving the - * patha. The instance name is the last element in the path + * path. The instance name is the last element in the path * @return a possibly empty map of application instance names to paths */ public Map<String, Path> listPersistentInstances() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 1e3379c..f282031 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; @@ -106,6 +107,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; /** * These are slider-specific Util methods @@ -1713,7 +1716,58 @@ public final class SliderUtils { } /** - * This wrapps ApplicationReports and generates a string version + * Given a source folder create zipped file + * + * @param srcFolder + * @param zipFile + * + * @throws IOException + */ + public static void zipFolder(File srcFolder, File zipFile) throws IOException { + log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath()); + List<String> files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true); + + byte[] buffer = new byte[1024]; + + try (FileOutputStream fos = new FileOutputStream(zipFile)) { + try (ZipOutputStream zos = new ZipOutputStream(fos)) { + + for (String file : files) { + ZipEntry ze = new ZipEntry(file); + zos.putNextEntry(ze); + try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) { + int len; + while ((len = in.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } + } + } + } + } + + + private static void generateFileList(List<String> fileList, File node, File rootFolder, Boolean relative) { + if (node.isFile()) { + String fileFullPath = node.toString(); + if (relative) { + fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, fileFullPath.length())); + } else { + fileList.add(fileFullPath); + } + } + + if (node.isDirectory()) { + String[] subNode = node.list(); + for (String filename : subNode) { + generateFileList(fileList, new File(node, filename), rootFolder, relative); + } + } + } + + /** + * This wraps ApplicationReports and generates a string version * iff the toString() operator is invoked */ public static class OnDemandReportStringifier { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 6812613..0a8dcdb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -49,6 +49,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; +import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; +import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE; import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR; import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR; @@ -142,6 +144,10 @@ public class InstanceBuilder { instancePaths.generatedConfPath.toUri()); internalOps.set(INTERNAL_DATA_DIR_PATH, instancePaths.dataPath.toUri()); + internalOps.set(INTERNAL_APPDEF_DIR_PATH, + instancePaths.appDefPath.toUri()); + internalOps.set(INTERNAL_ADDONS_DIR_PATH, + instancePaths.addonsPath.toUri()); internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java index df1206d..3505ac3 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java @@ -34,6 +34,8 @@ public class InstancePaths { public final Path dataPath; public final Path tmpPath; public final Path tmpPathAM; + public final Path appDefPath; + public final Path addonsPath; public InstancePaths(Path instanceDir) { this.instanceDir = instanceDir; @@ -45,6 +47,8 @@ public class InstancePaths { dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME); tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX); tmpPathAM = new Path(tmpPath, SliderKeys.AM_DIR_PREFIX); + appDefPath = new Path(tmpPath, SliderKeys.APP_DEF_DIR); + addonsPath = new Path(tmpPath, SliderKeys.ADDONS_DIR); } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 94d4c97..42ac47d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -40,10 +40,10 @@ import org.apache.slider.core.launch.AbstractLauncher; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.providers.agent.application.metadata.Application; -import org.apache.slider.providers.agent.application.metadata.Component; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.providers.agent.application.metadata.MetainfoParser; +import org.apache.slider.providers.agent.application.metadata.json.Application; +import org.apache.slider.providers.agent.application.metadata.json.Component; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfo; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfoParser; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -151,6 +151,7 @@ public class AgentClientProvider extends AbstractClientProvider } catch (BadConfigException bce) { throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); } + String appDef = instanceDefinition.getAppConfOperations(). getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); log.info("Validating app definition {}", appDef); @@ -163,13 +164,13 @@ public class AgentClientProvider extends AbstractClientProvider names.remove(SliderKeys.COMPONENT_AM); Map<Integer, String> priorityMap = new HashMap<Integer, String>(); - Metainfo metaInfo = null; + MetaInfo metaInfo = null; if (fs != null) { try { - metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef); + metaInfo = AgentUtils.getApplicationMetaInfo(fs, appDef); } catch (IOException ioe) { // Ignore missing metainfo file for now - log.info("Missing metainfo.xml {}", ioe.getMessage()); + log.info("Missing metainfo {}", ioe.getMessage()); } } @@ -217,8 +218,8 @@ public class AgentClientProvider extends AbstractClientProvider MapOperations componentConfig = resources.getMandatoryComponent(name); int count = componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - int definedMinCount = componentDef.getMinInstanceCountInt(); - int definedMaxCount = componentDef.getMaxInstanceCountInt(); + int definedMinCount = componentDef.getMinInstanceCount(); + int definedMaxCount = componentDef.getMaxInstanceCount(); if (count < definedMinCount || count > definedMaxCount) { throw new BadConfigException("Component %s, %s value %d out of range. " + "Expected minimum is %d and maximum is %d", @@ -264,9 +265,9 @@ public class AgentClientProvider extends AbstractClientProvider public Set<String> getApplicationTags(SliderFileSystem fileSystem, String appDef) throws SliderException { Set<String> tags; - Metainfo metainfo; + MetaInfo metainfo; try { - metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef); + metainfo = AgentUtils.getApplicationMetaInfo(fileSystem, appDef); } catch (IOException e) { log.error("Error retrieving metainfo from {}", appDef, e); throw new SliderException("Error retrieving metainfo", e); @@ -305,7 +306,7 @@ public class AgentClientProvider extends AbstractClientProvider File cmdDir = new File(tmpDir, "command"); cmdDir.mkdir(); - Metainfo metaInfo = null; + MetaInfo metaInfo = null; JSONObject defaultConfig = null; try { // expand app package into /app_pkg @@ -325,7 +326,7 @@ public class AgentClientProvider extends AbstractClientProvider while (offset < size) { offset += zipInputStream.read(content, offset, size - offset); } - metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content)); + metaInfo = new MetaInfoParser().fromInputStream(new ByteArrayInputStream(content)); } } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { int size = (int) zipEntry.getSize(); @@ -501,7 +502,7 @@ public class AgentClientProvider extends AbstractClientProvider protected JSONObject getCommandJson(JSONObject defaultConfig, JSONObject inputConfig, - Metainfo metainfo, + MetaInfo metainfo, File clientInstallPath) throws SliderException { try { JSONObject pkgList = new JSONObject(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 7977181..94dca51 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -67,6 +67,8 @@ public interface AgentKeys { String AGENT_MAIN_SCRIPT = "agent/main.py"; String APP_DEF = "application.def"; + String ADDON_PREFIX = "application.addon."; + String ADDONS = "application.addons"; String AGENT_VERSION = "agent.version"; String AGENT_CONF = "agent.conf"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 60236a5..4104cdd 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -66,17 +66,16 @@ import org.apache.slider.providers.AbstractProviderService; import org.apache.slider.providers.ProviderCore; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.providers.agent.application.metadata.Application; -import org.apache.slider.providers.agent.application.metadata.CommandScript; -import org.apache.slider.providers.agent.application.metadata.Component; -import org.apache.slider.providers.agent.application.metadata.ComponentExport; -import org.apache.slider.providers.agent.application.metadata.ConfigFile; +import org.apache.slider.providers.agent.application.metadata.json.Application; +import org.apache.slider.providers.agent.application.metadata.json.CommandScript; +import org.apache.slider.providers.agent.application.metadata.json.Component; +import org.apache.slider.providers.agent.application.metadata.json.ComponentCommand; +import org.apache.slider.providers.agent.application.metadata.json.ConfigFile; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; -import org.apache.slider.providers.agent.application.metadata.Export; -import org.apache.slider.providers.agent.application.metadata.ExportGroup; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.providers.agent.application.metadata.OSPackage; -import org.apache.slider.providers.agent.application.metadata.OSSpecific; +import org.apache.slider.providers.agent.application.metadata.json.Export; +import org.apache.slider.providers.agent.application.metadata.json.ExportGroup; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfo; +import org.apache.slider.providers.agent.application.metadata.json.Package; import org.apache.slider.providers.agent.application.metadata.PropertyInfo; import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss; import org.apache.slider.server.appmaster.actions.RegisterComponentInstance; @@ -156,7 +155,7 @@ public class AgentProviderService extends AbstractProviderService implements private int heartbeatMonitorInterval = 0; private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); - private volatile Metainfo metainfo = null; + private volatile MetaInfo metaInfo = null; private Map<String, DefaultConfig> defaultConfigs = null; private ComponentCommandOrder commandOrder = null; private HeartbeatMonitor monitor; @@ -232,7 +231,7 @@ public class AgentProviderService extends AbstractProviderService implements Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); for (String name : names) { - Component componentDef = getMetainfo().getApplicationComponent(name); + Component componentDef = getMetaInfo().getApplicationComponent(name); if (componentDef == null) { throw new BadConfigException( "Component %s is not a member of application.", name); @@ -241,8 +240,8 @@ public class AgentProviderService extends AbstractProviderService implements MapOperations componentConfig = resources.getMandatoryComponent(name); int count = componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - int definedMinCount = componentDef.getMinInstanceCountInt(); - int definedMaxCount = componentDef.getMaxInstanceCountInt(); + int definedMinCount = componentDef.getMinInstanceCount(); + int definedMaxCount = componentDef.getMaxInstanceCount(); if (count < definedMinCount || count > definedMaxCount) { throw new BadConfigException("Component %s, %s value %d out of range. " + "Expected minimum is %d and maximum is %d", @@ -261,21 +260,21 @@ public class AgentProviderService extends AbstractProviderService implements String appDef = instanceDefinition.getAppConfOperations() .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); - if (metainfo == null) { + if (metaInfo == null) { synchronized (syncLock) { - if (metainfo == null) { + if (metaInfo == null) { readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); - metainfo = getApplicationMetainfo(fileSystem, appDef); - if (metainfo == null || metainfo.getApplication() == null) { + metaInfo = getApplicationMetainfo(fileSystem, appDef); + if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( "metainfo.xml is required in app package."); } - commandOrder = new ComponentCommandOrder(metainfo.getApplication() - .getCommandOrder()); - defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo); + commandOrder = new ComponentCommandOrder(metaInfo.getApplication() + .getCommandOrders()); + defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); } @@ -715,15 +714,20 @@ public class AgentProviderService extends AbstractProviderService implements StateAccessForProviders accessor = getAmState(); CommandScript cmdScript = getScriptPathFromMetainfo(roleName); + List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands(); - if (cmdScript == null || cmdScript.getScript() == null) { + if ((cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) { log.error("role.script is unavailable for {}. Commands will not be sent.", roleName); return response; } - String scriptPath = cmdScript.getScript(); - long timeout = cmdScript.getTimeout(); + String scriptPath = null; + long timeout = 600L; + if(cmdScript != null) { + scriptPath = cmdScript.getScript(); + timeout = cmdScript.getTimeout(); + } if (timeout == 0L) { timeout = 600L; @@ -775,18 +779,49 @@ public class AgentProviderService extends AbstractProviderService implements try { if (Command.NOP != command) { if (command == Command.INSTALL) { - log.info("Installing {} on {}.", roleName, containerId); - addInstallCommand(roleName, containerId, response, scriptPath, timeout); - componentStatus.commandIssued(command); - } else if (command == Command.START) { - // check against dependencies - boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); - if (canExecute) { - log.info("Starting {} on {}.", roleName, containerId); - addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName)); + if(scriptPath != null) { + log.info("Installing {} on {}.", roleName, containerId); + addInstallCommand(roleName, containerId, response, scriptPath, timeout); componentStatus.commandIssued(command); } else { - log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); + // commands + log.info("Installing {} on {}.", roleName, containerId); + ComponentCommand installCmd = null; + for(ComponentCommand compCmd : commands) { + if(compCmd.getName().equals("INSTALL")) { + installCmd = compCmd; + } + } + addInstallCommand2(roleName, containerId, response, installCmd, timeout); + componentStatus.commandIssued(command); + } + } else if (command == Command.START) { + if(scriptPath != null) { + // check against dependencies + boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); + if (canExecute) { + log.info("Starting {} on {}.", roleName, containerId); + addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName)); + componentStatus.commandIssued(command); + } else { + log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); + } + } else { + // check against dependencies + boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); + if (canExecute) { + log.info("Starting {} on {}.", roleName, containerId); + ComponentCommand startCmd = null; + for(ComponentCommand compCmd : commands) { + if(compCmd.getName().equals("START")) { + startCmd = compCmd; + } + } + addStartCommand2(roleName, containerId, response, startCmd, timeout, false); + componentStatus.commandIssued(command); + } else { + log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); + } } } } @@ -856,8 +891,6 @@ public class AgentProviderService extends AbstractProviderService implements } } - // component specific publishes - processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName); processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName); // and update registration entries @@ -1010,8 +1043,8 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected Metainfo getMetainfo() { - return this.metainfo; + protected MetaInfo getMetaInfo() { + return this.metaInfo; } @VisibleForTesting @@ -1020,9 +1053,9 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException { - return AgentUtils.getApplicationMetainfo(fileSystem, appDef); + protected MetaInfo getApplicationMetainfo(SliderFileSystem fileSystem, + String appDef) throws IOException, BadConfigException { + return AgentUtils.getApplicationMetaInfo(fileSystem, appDef); } @VisibleForTesting @@ -1042,8 +1075,8 @@ public class AgentProviderService extends AbstractProviderService implements * @throws IOException */ protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem, - String appDef, Metainfo metainfo) throws IOException { - Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>(); + String appDef, MetaInfo metainfo) throws IOException { + Map<String, DefaultConfig> defaultConfigMap = new HashMap<>(); if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) { for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) { DefaultConfig config = null; @@ -1224,7 +1257,7 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Status report: {}", status.toString()); if (status.getConfigs() != null) { - Application application = getMetainfo().getApplication(); + Application application = getMetaInfo().getApplication(); if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) { // If no Master can explicitly publish then publish if its a master @@ -1359,56 +1392,6 @@ public class AgentProviderService extends AbstractProviderService implements } } - /** Publish component instance specific data if the component demands it */ - protected void processAndPublishComponentSpecificData(Map<String, String> ports, - String containerId, - String hostFqdn, - String componentName) { - String portVarFormat = "${site.%s}"; - String hostNamePattern = "${THIS_HOST}"; - Map<String, String> toPublish = new HashMap<String, String>(); - - Application application = getMetainfo().getApplication(); - for (Component component : application.getComponents()) { - if (component.getName().equals(componentName)) { - if (!component.getComponentExports().isEmpty()) { - - for (ComponentExport export : component.getComponentExports()) { - String templateToExport = export.getValue(); - for (String portName : ports.keySet()) { - boolean publishData = false; - String portValPattern = String.format(portVarFormat, portName); - if (templateToExport.contains(portValPattern)) { - templateToExport = templateToExport.replace(portValPattern, ports.get(portName)); - publishData = true; - } - if (templateToExport.contains(hostNamePattern)) { - templateToExport = templateToExport.replace(hostNamePattern, hostFqdn); - publishData = true; - } - if (publishData) { - toPublish.put(export.getName(), templateToExport); - log.info("Publishing {} for name {} and container {}", - templateToExport, export.getName(), containerId); - } - } - } - } - } - } - - if (!toPublish.isEmpty()) { - Map<String, String> perContainerData = null; - if (!getComponentInstanceData().containsKey(containerId)) { - perContainerData = new ConcurrentHashMap<String, String>(); - } else { - perContainerData = getComponentInstanceData().get(containerId); - } - perContainerData.putAll(toPublish); - getComponentInstanceData().put(containerId, perContainerData); - publishComponentInstanceData(); - } - } /** Publish component instance specific data if the component demands it */ protected void processAndPublishComponentSpecificExports(Map<String, String> ports, @@ -1418,8 +1401,8 @@ public class AgentProviderService extends AbstractProviderService implements String portVarFormat = "${site.%s}"; String hostNamePattern = "${" + compName + "_HOST}"; - List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups(); - Component component = getMetainfo().getApplicationComponent(compName); + List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups(); + Component component = getMetaInfo().getApplicationComponent(compName); if (component != null && SliderUtils.isSet(component.getCompExports()) && SliderUtils.isNotEmpty(appExportGroups)) { @@ -1521,7 +1504,7 @@ public class AgentProviderService extends AbstractProviderService implements * @return the component entry or null for no match */ protected Component getApplicationComponent(String roleName) { - return getMetainfo().getApplicationComponent(roleName); + return getMetaInfo().getApplicationComponent(roleName); } /** @@ -1581,7 +1564,7 @@ public class AgentProviderService extends AbstractProviderService implements protected boolean isMarkedAutoRestart(String roleName) { Component component = getApplicationComponent(roleName); if (component != null) { - return component.getRequiresAutoRestart(); + return component.getAutoStartOnFailure(); } return false; } @@ -1593,7 +1576,7 @@ public class AgentProviderService extends AbstractProviderService implements */ protected boolean canAnyMasterPublishConfig() { if (canAnyMasterPublish == null) { - Application application = getMetainfo().getApplication(); + Application application = getMetaInfo().getApplication(); if (application == null) { log.error("Malformed app definition: Expect application as root element in the metainfo.xml"); } else { @@ -1663,20 +1646,53 @@ public class AgentProviderService extends AbstractProviderService implements response.addExecutionCommand(cmd); } + @VisibleForTesting + protected void addInstallCommand2(String componentName, + String containerId, + HeartBeatResponse response, + ComponentCommand compCmd, + long timeout) + throws SliderException { + assert getAmState().isApplicationLive(); + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); + + ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); + prepareExecutionCommand(cmd); + String clusterName = getClusterName(); + cmd.setClusterName(clusterName); + cmd.setRoleCommand(Command.INSTALL.toString()); + cmd.setServiceName(clusterName); + cmd.setComponentName(componentName); + cmd.setRole(componentName); + Map<String, String> hostLevelParams = new TreeMap<String, String>(); + hostLevelParams.put(PACKAGE_LIST, getPackageList()); + hostLevelParams.put(CONTAINER_ID, containerId); + cmd.setHostLevelParams(hostLevelParams); + + Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName); + cmd.setConfigurations(configurations); + + ComponentCommand effectiveCommand = compCmd; + if(compCmd == null) { + effectiveCommand = new ComponentCommand(); + effectiveCommand.setName("INSTALL"); + effectiveCommand.setExec("DEFAULT"); + } + cmd.setCommandParams(setCommandParameters(effectiveCommand, timeout, false)); + configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); + + cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); + response.addExecutionCommand(cmd); + } + protected static String getPackageListFromApplication(Application application) { String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}"; String pkgListFormatString = "[%s]"; List<String> packages = new ArrayList(); if (application != null) { - List<OSSpecific> osSpecifics = application.getOSSpecifics(); - if (osSpecifics != null && osSpecifics.size() > 0) { - for (OSSpecific osSpecific : osSpecifics) { - if (osSpecific.getOsType().equals("any")) { - for (OSPackage osPackage : osSpecific.getPackages()) { - packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName())); - } - } - } + List<Package> appPackages = application.getPackages(); + for (Package appPackage : appPackages){ + packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName())); } } @@ -1688,7 +1704,7 @@ public class AgentProviderService extends AbstractProviderService implements } private String getPackageList() { - return getPackageListFromApplication(getMetainfo().getApplication()); + return getPackageListFromApplication(getMetaInfo().getApplication()); } private void prepareExecutionCommand(ExecutionCommand cmd) { @@ -1708,6 +1724,18 @@ public class AgentProviderService extends AbstractProviderService implements return cmdParams; } + private Map<String, String> setCommandParameters(ComponentCommand compCmd, long timeout, boolean recordConfig) { + Map<String, String> cmdParams = new TreeMap<String, String>(); + cmdParams.put("service_package_folder", + "${AGENT_WORK_ROOT}/work/app/definition/package"); + cmdParams.put("command", compCmd.getExec()); + cmdParams.put("schema_version", "2.0"); + cmdParams.put("command_timeout", Long.toString(timeout)); + cmdParams.put("script_type", compCmd.getType()); + cmdParams.put("record_config", Boolean.toString(recordConfig)); + return cmdParams; + } + @VisibleForTesting protected void addStatusCommand(String componentName, String containerId, @@ -1830,6 +1858,42 @@ public class AgentProviderService extends AbstractProviderService implements response.addExecutionCommand(cmdStop); } + @VisibleForTesting + protected void addStartCommand2(String componentName, String containerId, HeartBeatResponse response, + ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart) + throws + SliderException { + assert getAmState().isApplicationLive(); + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); + ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); + + ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); + prepareExecutionCommand(cmd); + String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); + String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); + cmd.setHostname(hostName); + cmd.setClusterName(clusterName); + cmd.setRoleCommand(Command.START.toString()); + cmd.setServiceName(clusterName); + cmd.setComponentName(componentName); + cmd.setRole(componentName); + Map<String, String> hostLevelParams = new TreeMap<>(); + hostLevelParams.put(CONTAINER_ID, containerId); + cmd.setHostLevelParams(hostLevelParams); + + Map<String, String> roleParams = new TreeMap<>(); + cmd.setRoleParams(roleParams); + cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart)); + + cmd.setCommandParams(setCommandParameters(startCommand, timeout, true)); + + Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName); + + cmd.setConfigurations(configurations); + configurations.get("global").put("exec_cmd", startCommand.getExec()); + response.addExecutionCommand(cmd); + } + protected Map<String, String> getAllocatedPorts() { return getAllocatedPorts(SHARED_PORT_TAG); } @@ -1934,7 +1998,7 @@ public class AgentProviderService extends AbstractProviderService implements List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); - List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles(); + List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles(); for (ConfigFile configFile : configFiles) { log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index 1d61c15..2795692 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -16,14 +16,18 @@ */ package org.apache.slider.providers.agent; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser; import org.apache.slider.providers.agent.application.metadata.Metainfo; import org.apache.slider.providers.agent.application.metadata.MetainfoParser; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfo; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfoParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +35,7 @@ import java.io.FileNotFoundException; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.StringWriter; /** * @@ -38,22 +43,33 @@ import java.io.InputStream; public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); - public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException { + public static MetaInfo getApplicationMetaInfo(SliderFileSystem fileSystem, + String appDef) throws IOException, BadConfigException { log.info("Reading metainfo at {}", appDef); FileSystem fs = fileSystem.getFileSystem(); Path appPath = new Path(appDef); - InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream( - fs, appPath, "metainfo.xml"); - if (metainfoStream == null) { - log.error("metainfo.xml is unavailable at {}.", appDef); - throw new FileNotFoundException("metainfo.xml is required in app package. " + - appPath); - } - Metainfo metainfo = new MetainfoParser().parse(metainfoStream); + MetaInfo metaInfo = null; + InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( + fs, appPath, "metainfo.json"); + if (metainfoJsonStream == null) { + InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream( + fs, appPath, "metainfo.xml"); + if (metainfoXMLStream != null) { + Metainfo metaInfoFromXML = new MetainfoParser().parse(metainfoXMLStream); + metaInfo = MetaInfo.upConverted(metaInfoFromXML); + } + } else { + MetaInfoParser parser = new MetaInfoParser(); + metaInfo = parser.fromInputStream(metainfoJsonStream); + } - return metainfo; + if (metaInfo == null) { + log.error("metainfo is unavailable at {}.", appDef); + throw new FileNotFoundException("metainfo.xml/json is required in app package. " + + appPath); + } + return metaInfo; } static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index f4ace5f..8e41b6b 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -18,7 +18,7 @@ package org.apache.slider.providers.agent; -import org.apache.slider.providers.agent.application.metadata.CommandOrder; +import org.apache.slider.providers.agent.application.metadata.json.CommandOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,13 +49,13 @@ public class ComponentCommandOrder { if (requiredStates.size() > 0) { Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command); if (compDep == null) { - compDep = new HashMap<String, List<ComponentState>>(); + compDep = new HashMap<>(); dependencies.put(componentCmd.command, compDep); } List<ComponentState> requirements = compDep.get(componentCmd.componentName); if (requirements == null) { - requirements = new ArrayList<ComponentState>(); + requirements = new ArrayList<>(); compDep.put(componentCmd.componentName, requirements); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index b34cba1..ea23678 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -16,6 +16,7 @@ */ package org.apache.slider.providers.agent.application.metadata; +import org.apache.slider.providers.agent.application.metadata.json.MetaInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,17 @@ public class Metainfo { String schemaVersion; Application application; + MetaInfo jsonEncoded; public Metainfo() { } + // TODO: Temporary code - the plan is to move to the json parsed MetaInfo + public Metainfo(MetaInfo jsonEncoded) { + jsonEncoded = jsonEncoded; + // initialize the class + } + public String getSchemaVersion() { return schemaVersion; } @@ -60,4 +68,8 @@ public class Metainfo { } return null; } + + public static Metainfo fromJsonObject(MetaInfo jsonEncoded) { + return new Metainfo(jsonEncoded); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Application.java new file mode 100644 index 0000000..4dfa18d --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Application.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata.json; + +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Represents the metadata associated with the application. + */ +public class Application { + protected static final Logger + log = LoggerFactory.getLogger(Application.class); + + + private String name; + private List<Component> components = new ArrayList<>(); + private String comment; + private String version; + private String exportedConfigs; + private List<ExportGroup> exportGroups = new ArrayList<>(); + private List<Package> packages = new ArrayList<>(); + private List<CommandOrder> commandOrders = new ArrayList<>(); + private List<ConfigFile> configFiles = new ArrayList<>(); + + public Application() { + } + + @JsonProperty("components") + public List<Component> getComponents() { + return this.components; + } + + @JsonProperty("exportGroups") + public List<ExportGroup> getExportGroups() { + return this.exportGroups; + } + + @JsonProperty("packages") + public List<Package> getPackages() { + return this.packages; + } + + @JsonProperty("commandOrders") + public List<CommandOrder> getCommandOrders() { + return this.commandOrders; + } + + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return this.configFiles; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @JsonIgnore + public Component getApplicationComponent(String name) { + for (Component component : components) { + if (name.equals(component.getName())) { + return component; + } + } + return null; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getExportedConfigs() { + return exportedConfigs; + } + + public void setExportedConfigs(String exportedConfigs) { + this.exportedConfigs = exportedConfigs; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandOrder.java new file mode 100644 index 0000000..1ac12aa --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandOrder.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata.json; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents package description. + */ +public class CommandOrder { + protected static final Logger + log = LoggerFactory.getLogger(CommandOrder.class); + + + private String command; + private String requires; + + /** + * Creator. + */ + public CommandOrder() { + } + + + public void setCommand(String name) { + command = command; + } + + public String getCommand() { + return command; + } + + public void setRequires(String type) { + requires = requires; + } + + public String getRequires() { + return requires; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandScript.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandScript.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandScript.java new file mode 100644 index 0000000..51869a0 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/CommandScript.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata.json; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents package description. + */ +public class CommandScript { + protected static final Logger + log = LoggerFactory.getLogger(CommandScript.class); + + + private String script; + private String scriptType; + private long timeout; + + /** + * Creator. + */ + public CommandScript() { + } + + + public void setScript(String script) { + this.script = script; + } + + public String getScript() { + return script; + } + + public String getScriptType() { + return scriptType; + } + + public void setScriptType(String scriptType) { + this.scriptType = scriptType; + } + + public long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a31d20ee/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Component.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Component.java new file mode 100644 index 0000000..8697e48 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/json/Component.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata.json; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents the metadata associated with the application. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class Component { + protected static final Logger + log = LoggerFactory.getLogger(Component.class); + + private String name; + private String compExports; + private String category = "MASTER"; + private String publishConfig; + private int minInstanceCount = 1; + private int maxInstanceCount = Integer.MAX_VALUE; + private Boolean autoStartOnFailure = false; + private String appExports; + private CommandScript commandScript; + private List<ComponentCommand> commands = new ArrayList<ComponentCommand>(); + + public Component() { + } + + @JsonProperty("commands") + public List<ComponentCommand> getCommands() { + return this.commands; + } + + public CommandScript getCommandScript() { + return commandScript; + } + + public void setCommandScript(CommandScript commandScript) { + this.commandScript = commandScript; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getCompExports() { + return compExports; + } + + public void setCompExports(String compExports) { + this.compExports = compExports; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getPublishConfig() { + return publishConfig; + } + + public void setPublishConfig(String publishConfig) { + this.publishConfig = publishConfig; + } + + public int getMinInstanceCount() { + return minInstanceCount; + } + + public void setMinInstanceCount(int minInstanceCount) { + this.minInstanceCount = minInstanceCount; + } + + public int getMaxInstanceCount() { + return maxInstanceCount; + } + + public void setMaxInstanceCount(int maxInstanceCount) { + this.maxInstanceCount = maxInstanceCount; + } + + public Boolean getAutoStartOnFailure() { + return autoStartOnFailure; + } + + public void setAutoStartOnFailure(Boolean autoStartOnFailure) { + this.autoStartOnFailure = autoStartOnFailure; + } + + public String getAppExports() { + return appExports; + } + + public void setAppExports(String appExports) { + this.appExports = appExports; + } +}
