http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java new file mode 100644 index 0000000..78bb8c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.SliderException; +import org.codehaus.jackson.annotate.JsonProperty; +import java.util.ArrayList; +import java.util.List; + +/** + * Component defined in master package metainfo.json + */ +public class Component extends AbstractComponent { + + String category = CATEGORY_MASTER; + String publishConfig = Boolean.FALSE.toString(); + String minInstanceCount = "0"; + String maxInstanceCount; + String autoStartOnFailure = Boolean.FALSE.toString(); + String appExports; + String compExports; + String type = TYPE_STANDARD; + List<ComponentExport> componentExports = new ArrayList<>(); + List<DockerContainer> dockerContainers = new ArrayList<>(); + List<ConfigFile> configFiles = new ArrayList<>(); + + public Component() { + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getPublishConfig() { + return publishConfig; + } + + public void setPublishConfig(String publishConfig) { + this.publishConfig = publishConfig; + } + + public String getAutoStartOnFailure() { + return autoStartOnFailure; + } + + public void setAutoStartOnFailure(String autoStartOnFailure) { + this.autoStartOnFailure = autoStartOnFailure; + } + + public String getAppExports() { + return appExports; + } + + public void setAppExports(String appExports) { + this.appExports = appExports; + } + + public String getCompExports() { + return compExports; + } + + public void setCompExports(String compExports) { + this.compExports = compExports; + } + + public String getMinInstanceCount() { + return minInstanceCount; + } + + @JsonProperty("dockerContainers") + public List<DockerContainer> getDockerContainers() { + return this.dockerContainers; + } + + public Boolean getAutoStartOnFailureBoolean() { + if (SliderUtils.isUnset(getAutoStartOnFailure())) { + return Boolean.FALSE; + } + + return Boolean.parseBoolean(getAutoStartOnFailure()); + } + + public int getMinInstanceCountInt() throws BadConfigException { + if (SliderUtils.isUnset(minInstanceCount)) { + return 0; + } + + try { + return Integer.parseInt(minInstanceCount); + } catch (NumberFormatException nfe) { + throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name); + } + } + + public int getMaxInstanceCountInt() throws BadConfigException { + if (SliderUtils.isUnset(maxInstanceCount)) { + return Integer.MAX_VALUE; + } + + try { + return Integer.parseInt(maxInstanceCount); + } catch (NumberFormatException nfe) { + throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name); + } + } + + public void setMinInstanceCount(String minInstanceCount) { + this.minInstanceCount = minInstanceCount; + } + + public String getMaxInstanceCount() { + return maxInstanceCount; + } + + public void setMaxInstanceCount(String maxInstanceCount) { + this.maxInstanceCount = maxInstanceCount; + } + + public void addComponentExport(ComponentExport export) { + componentExports.add(export); + } + + public List<ComponentExport> getComponentExports() { + return componentExports; + } + + public Boolean getRequiresAutoRestart() { + return Boolean.parseBoolean(this.autoStartOnFailure); + } + + public void addConfigFile(ConfigFile configFile) { + this.configFiles.add(configFile); + } + + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append("\n\"name\": ").append(name); + sb.append(",\n\"category\": ").append(category); + sb.append(",\n\"commandScript\" :").append(commandScript); + for(DockerContainer dc : dockerContainers){ + sb.append(",\n\"container\" :").append(dc.toString()); + } + sb.append('}'); + return sb.toString(); + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "component"); + Metainfo.checkNonNull(getCategory(), "category", "component"); + if (!getCategory().equals(CATEGORY_MASTER) + && !getCategory().equals(CATEGORY_SLAVE) + && !getCategory().equals(CATEGORY_CLIENT)) { + throw new SliderException("Invalid category for the component " + getCategory()); + } + + Metainfo.checkNonNull(getType(), "type", "component"); + if (!getType().equals(TYPE_DOCKER) + && !getType().equals(TYPE_STANDARD)) { + throw new SliderException("Invalid type for the component " + getType()); + } + + if (version.equals(Metainfo.VERSION_TWO_ZERO)) { + if (getType().equals(TYPE_DOCKER)) { + throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO); + } + + if (getCommands().size() > 0) { + throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO); + } + } + + if (commandScript != null) { + commandScript.validate(version); + } + + if (version.equals(Metainfo.VERSION_TWO_ONE)) { + for (ComponentCommand cc : getCommands()) { + cc.validate(version); + } + } + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java new file mode 100644 index 0000000..52117c5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents the metadata associated with the application. + */ +public class ComponentCommand implements Validate { + protected static final Logger + log = LoggerFactory.getLogger(ComponentCommand.class); + + + private String exec; + private String name = "START"; + private String type = "SHELL"; + + /** + * Creator. + */ + public ComponentCommand() { + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setExec(String exec) { + this.exec = exec; + } + + public String getExec() { + return exec; + } + + public void setType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "componentCommand"); + + Metainfo.checkNonNull(getType(), "version", "application"); + } + + public static ComponentCommand getDefaultComponentCommand() { + ComponentCommand cc = new ComponentCommand(); + cc.setExec("DEFAULT"); + return cc; + } + + public static ComponentCommand getDefaultComponentCommand(String commandName) { + ComponentCommand cc = new ComponentCommand(); + cc.setExec("DEFAULT"); + cc.setName(commandName); + return cc; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java new file mode 100644 index 0000000..a18854c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +/** + * + */ +public class ComponentExport { + String name; + String value; + + public ComponentExport() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"value\": ").append(value); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java new file mode 100644 index 0000000..855e5b6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +public class ComponentsInAddonPackage extends AbstractComponent { + + @Override + public void validate(String version) throws SliderException { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java new file mode 100644 index 0000000..cb47512 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +/** + * + */ +public class ConfigFile implements Validate { + String type; + String fileName; + String dictionaryName; + + public ConfigFile() { + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getDictionaryName() { + return dictionaryName; + } + + public void setDictionaryName(String dictionaryName) { + this.dictionaryName = dictionaryName; + } + + public void validate(String version) throws SliderException { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java new file mode 100644 index 0000000..46c8836 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import java.util.ArrayList; +import java.util.List; + +/** + * Application default config + */ +public class DefaultConfig { + List<PropertyInfo> propertyInfos; + + public DefaultConfig() { + propertyInfos = new ArrayList<PropertyInfo>(); + } + + public void addPropertyInfo(PropertyInfo propertyInfo) { + propertyInfos.add(propertyInfo); + } + + public List<PropertyInfo> getPropertyInfos() { + return propertyInfos; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java new file mode 100644 index 0000000..e136775 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.commons.digester.Digester; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.InputStream; + +/** + * + */ +public class DefaultConfigParser { + + public DefaultConfig parse(InputStream configFileStream) throws IOException { + Digester digester = new Digester(); + digester.setValidating(false); + + digester.addObjectCreate("configuration", DefaultConfig.class); + + digester.addObjectCreate("*/property", PropertyInfo.class); + digester.addBeanPropertySetter("*/property/name"); + digester.addBeanPropertySetter("*/property/value"); + digester.addBeanPropertySetter("*/property/description"); + digester.addSetNext("*/property", "addPropertyInfo"); + + try { + return (DefaultConfig) digester.parse(configFileStream); + } catch (IOException e) { + + } catch (SAXException e) { + + } finally { + configFileStream.close(); + } + + return null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java new file mode 100644 index 0000000..4c61e7f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; +import org.codehaus.jackson.annotate.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a docker container + */ +public class DockerContainer implements Validate { + protected static final Logger + log = LoggerFactory.getLogger(DockerContainer.class); + + private String name; + private String image; + private String network; + private String useNetworkScript; + private String options; + private List<DockerContainerMount> mounts = new ArrayList<>(); + private List<DockerContainerPort> ports = new ArrayList<>(); + private String statusCommand; + private String startCommand; + private String commandPath; + private String additionalParam; + private String runPrivilegedContainer; + private List<DockerContainerInputFile> inputFiles = new ArrayList<>(); + private List<ConfigFile> configFiles = new ArrayList<>(); + + public DockerContainer() { + } + + @JsonProperty("mounts") + public List<DockerContainerMount> getMounts() { return this.mounts; } + + @JsonProperty("ports") + public List<DockerContainerPort> getPorts() { + return this.ports; + } + + @JsonProperty("inputFiles") + public List<DockerContainerInputFile> getInputFiles() { + return this.inputFiles; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public String getNetwork() { + return network; + } + + public void setNetwork(String network) { + this.network = network; + } + + public String getUseNetworkScript() { + return useNetworkScript; + } + + public void setUseNetworkScript(String useNetworkScript) { + this.useNetworkScript = useNetworkScript; + } + + public String getOptions() { + return options; + } + + public void setOptions(String options) { + this.options = options; + } + + @Override + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "dockerContainer"); + Metainfo.checkNonNull(getImage(), "image", "dockerContainer"); + for (DockerContainerMount dcm : getMounts()) { + dcm.validate(version); + } + for (DockerContainerPort dcp : getPorts()) { + dcp.validate(version); + } + } + + @JsonProperty("statusCommand") + public String getStatusCommand() { + return statusCommand; + } + + @JsonProperty("statusCommand") + public void setStatusCommand(String statusCommand) { + this.statusCommand = statusCommand; + } + + public String getCommandPath() { + return commandPath; + } + + public void setCommandPath(String commandPath) { + this.commandPath = commandPath; + } + + public String getAdditionalParam() { + return additionalParam; + } + + public void setAdditionalParam(String additionalParam) { + this.additionalParam = additionalParam; + } + + @JsonProperty("startCommand") + public String getStartCommand() { + return startCommand; + } + + @JsonProperty("startCommand") + public void setStartCommand(String startCommand) { + this.startCommand = startCommand; + } + + @JsonProperty("runPrivilegedContainer") + public String getRunPrivilegedContainer() { + return runPrivilegedContainer; + } + + @JsonProperty("runPrivilegedContainer") + public void setRunPrivilegedContainer(String runPrivilegedContainer) { + this.runPrivilegedContainer = runPrivilegedContainer; + } + + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + + public void setConfigFiles(List<ConfigFile> configFiles) { + this.configFiles = configFiles; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder("DockerContainer [name=") + .append(name).append(", image=").append(image).append(", options=") + .append(options).append(", mounts=").append(mounts).append(", ports=") + .append(ports).append(", statusCommand=").append(statusCommand) + .append(", commandPath=").append(commandPath) + .append(", additionalParam=").append(additionalParam) + .append(", inputFiles=").append(inputFiles).append(", startCommand=") + .append(startCommand).append(", runPriviledgedContainer=") + .append(runPrivilegedContainer).append(", configFiles=") + .append(configFiles).append("]"); + return result.toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java new file mode 100644 index 0000000..0faceb9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DockerContainerInputFile { + protected static final Logger log = LoggerFactory + .getLogger(DockerContainerInputFile.class); + + private String containerPath; + private String fileLocalPath; + + public DockerContainerInputFile() { + } + + public String getContainerMount() { + return containerPath; + } + + public void setContainerMount(String containerMount) { + this.containerPath = containerMount; + } + + public String getFileLocalPath() { + return fileLocalPath; + } + + public void setFileLocalPath(String fileLocalPath) { + this.fileLocalPath = fileLocalPath; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java new file mode 100644 index 0000000..61f07f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents a docker container mount + */ +public class DockerContainerMount implements Validate { + protected static final Logger + log = LoggerFactory.getLogger(DockerContainerMount.class); + + + private String containerMount; + private String hostMount; + + public DockerContainerMount() { + } + + public String getContainerMount() { + return containerMount; + } + + public void setContainerMount(String containerMount) { + this.containerMount = containerMount; + } + + public String getHostMount() { + return hostMount; + } + + public void setHostMount(String hostMount) { + this.hostMount = hostMount; + } + + @Override + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount"); + Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java new file mode 100644 index 0000000..0629d9d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents a docker container port + */ +public class DockerContainerPort implements Validate { + protected static final Logger + log = LoggerFactory.getLogger(DockerContainerPort.class); + + + private String containerPort; + private String hostPort; + + public DockerContainerPort() { + } + + public String getContainerPort() { + return containerPort; + } + + public void setContainerPort(String containerPort) { + this.containerPort = containerPort; + } + + public String getHostPort() { + return hostPort; + } + + public void setHostPort(String hostPort) { + this.hostPort = hostPort; + } + + @Override + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort"); + Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort"); + } + + @Override + public String toString() { + return "DockerContainerPort [containerPort=" + containerPort + + ", hostPort=" + hostPort + "]"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java new file mode 100644 index 0000000..5e0fb24 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +/** + * + */ +public class Export implements Validate { + String name; + String value; + + public Export() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"value\": ").append(value); + sb.append('}'); + return sb.toString(); + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "export"); + Metainfo.checkNonNull(getValue(), "value", "export"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java new file mode 100644 index 0000000..3d9f34c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class ExportGroup implements Validate { + String name; + List<Export> exports; + + public ExportGroup() { + exports = new ArrayList<Export>(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void addExport(Export export) { + exports.add(export); + } + + public List<Export> getExports() { + return exports; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("{"); + sb.append(",\n\"name\": ").append(name); + sb.append(",\n\"exports\" : {"); + for (Export export : exports) { + sb.append("\n").append(export); + } + sb.append("\n},"); + sb.append('}'); + return sb.toString(); + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "exportGroup"); + for(Export exp : getExports()) { + exp.validate(version); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java new file mode 100644 index 0000000..10c497f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Application metainfo uber class + */ +public class Metainfo { + protected static final Logger log = + LoggerFactory.getLogger(Metainfo.class); + public static String VERSION_TWO_ZERO = "2.0"; + public static String VERSION_TWO_ONE = "2.1"; + + String schemaVersion; + ApplicationPackage applicationPackage; + Application application; + + public String getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(String schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public ApplicationPackage getApplicationPackage() { + return applicationPackage; + } + + public void setApplicationPackage(ApplicationPackage pkg) { + this.applicationPackage = pkg; + } + + public Application getApplication() { + return application; + } + + public void setApplication(Application application) { + this.application = application; + } + + public Component getApplicationComponent(String roleGroup) { + if (application == null) { + log.error("Malformed app definition: Expect application as the top level element for metainfo"); + } else { + for (Component component : application.getComponents()) { + if (component.getName().equals(roleGroup)) { + return component; + } + } + } + return null; + } + + public List<ConfigFile> getComponentConfigFiles(String roleGroup) { + List<ConfigFile> componentConfigFiles = new ArrayList<>(); + componentConfigFiles.addAll(application.getConfigFiles()); + Component component = getApplicationComponent(roleGroup); + if (component != null) { + componentConfigFiles.addAll(component.getConfigFiles()); + } + return componentConfigFiles; + } + + public void validate() throws SliderException { + if (!VERSION_TWO_ONE.equals(schemaVersion) && + !VERSION_TWO_ZERO.equals(schemaVersion)) { + throw new SliderException("Unsupported version " + getSchemaVersion()); + } + if (application != null) { + application.validate(schemaVersion); + } + if (applicationPackage != null) { + applicationPackage.validate(schemaVersion); + } + } + + public static void checkNonNull(String value, String field, String type) throws SliderException { + if (SliderUtils.isUnset(value)) { + throw new SliderException(type + "." + field + " cannot be null"); + } + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Metainfo [schemaVersion="); + builder.append(schemaVersion); + builder.append(", applicationPackage="); + builder.append(applicationPackage); + builder.append(", application="); + builder.append(application); + builder.append("]"); + return builder.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java new file mode 100644 index 0000000..8b520eb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.commons.digester.Digester; + +/** + * + */ +public class MetainfoParser extends AbstractMetainfoParser{ + + protected void composeSchema(Digester digester){ + digester.addObjectCreate("metainfo", Metainfo.class); + digester.addBeanPropertySetter("metainfo/schemaVersion"); + + digester.addObjectCreate("*/application", Application.class); + digester.addBeanPropertySetter("*/application/name"); + digester.addBeanPropertySetter("*/application/comment"); + digester.addBeanPropertySetter("*/application/version"); + digester.addBeanPropertySetter("*/application/exportedConfigs"); + + digester.addObjectCreate("*/commandOrder", CommandOrder.class); + digester.addBeanPropertySetter("*/commandOrder/command"); + digester.addBeanPropertySetter("*/commandOrder/requires"); + digester.addSetNext("*/commandOrder", "addCommandOrder"); + + digester.addObjectCreate("*/exportGroup", ExportGroup.class); + digester.addBeanPropertySetter("*/exportGroup/name"); + digester.addObjectCreate("*/export", Export.class); + digester.addBeanPropertySetter("*/export/name"); + digester.addBeanPropertySetter("*/export/value"); + digester.addSetNext("*/export", "addExport"); + digester.addSetNext("*/exportGroup", "addExportGroup"); + + digester.addObjectCreate("*/component", Component.class); + digester.addBeanPropertySetter("*/component/name"); + digester.addBeanPropertySetter("*/component/category"); + digester.addBeanPropertySetter("*/component/publishConfig"); + digester.addBeanPropertySetter("*/component/minInstanceCount"); + digester.addBeanPropertySetter("*/component/maxInstanceCount"); + digester.addBeanPropertySetter("*/component/autoStartOnFailure"); + digester.addBeanPropertySetter("*/component/appExports"); + digester.addBeanPropertySetter("*/component/compExports"); + digester.addObjectCreate("*/componentExport", ComponentExport.class); + digester.addBeanPropertySetter("*/componentExport/name"); + digester.addBeanPropertySetter("*/componentExport/value"); + digester.addSetNext("*/componentExport", "addComponentExport"); + digester.addSetNext("*/component", "addComponent"); + + digester.addObjectCreate("*/commandScript", CommandScript.class); + digester.addBeanPropertySetter("*/commandScript/script"); + digester.addBeanPropertySetter("*/commandScript/scriptType"); + digester.addBeanPropertySetter("*/commandScript/timeout"); + digester.addSetNext("*/commandScript", "addCommandScript"); + + digester.addObjectCreate("*/command", ComponentCommand.class); + digester.addBeanPropertySetter("*/command/exec"); + digester.addBeanPropertySetter("*/command/name"); + digester.addBeanPropertySetter("*/command/type"); + digester.addSetNext("*/command", "addCommand"); + + digester.addObjectCreate("*/osSpecific", OSSpecific.class); + digester.addBeanPropertySetter("*/osSpecific/osType"); + digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class); + digester.addBeanPropertySetter("*/osSpecific/packages/package/type"); + digester.addBeanPropertySetter("*/osSpecific/packages/package/name"); + digester.addSetNext("*/osSpecific/packages/package", "addOSPackage"); + digester.addSetNext("*/osSpecific", "addOSSpecific"); + + digester.addObjectCreate("*/application/packages/package", Package.class); + digester.addBeanPropertySetter("*/application/packages/package/type"); + digester.addBeanPropertySetter("*/application/packages/package/name"); + digester.addSetNext("*/application/packages/package", "addPackage"); + + digester.addObjectCreate("*/configFile", ConfigFile.class); + digester.addBeanPropertySetter("*/configFile/type"); + digester.addBeanPropertySetter("*/configFile/fileName"); + digester.addBeanPropertySetter("*/configFile/dictionaryName"); + digester.addSetNext("*/configFile", "addConfigFile"); + + digester.addSetRoot("*/application", "setApplication"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java new file mode 100644 index 0000000..32b4890 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +/** + * + */ +public class OSPackage implements Validate { + String type; + String name; + + public OSPackage() { + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "osPackage"); + Metainfo.checkNonNull(getType(), "type", "osPackage"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java new file mode 100644 index 0000000..c06d498 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class OSSpecific implements Validate { + String osType; + List<OSPackage> packages; + + public OSSpecific() { + packages = new ArrayList<OSPackage>(); + } + + public String getOsType() { + return osType; + } + + public void setOsType(String osType) { + this.osType = osType; + } + + public void addOSPackage(OSPackage osPackage) { + packages.add(osPackage); + } + + public List<OSPackage> getPackages() { + return packages; + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getOsType(), "osType", "osSpecific"); + for (OSPackage opkg : getPackages()) { + opkg.validate(version); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java new file mode 100644 index 0000000..b88f77d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Represents package description. + */ +public class Package implements Validate { + protected static final Logger + log = LoggerFactory.getLogger(Package.class); + + + private String name; + private String type; + + public Package() { + } + + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getName(), "name", "package"); + Metainfo.checkNonNull(getType(), "type", "package"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java new file mode 100644 index 0000000..62ee0f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +/** + * Application config property info + */ +public class PropertyInfo { + String name; + String value; + String description; + + public PropertyInfo() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java new file mode 100644 index 0000000..ef03dcd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent.application.metadata; + +import org.apache.slider.core.exceptions.SliderException; + +/** + * Implementer provides a validate method + */ +public interface Validate { + + public void validate(String version) throws SliderException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md new file mode 100644 index 0000000..dfd1373 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md @@ -0,0 +1,22 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# todo list + +* Retry on failure + * Agent can toleate a configurable number of failures (e.g. 3) before giving up +* Agent should separate out hostname and label that is received for registration http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java new file mode 100644 index 0000000..e5430f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers.slideram; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.slider.api.InternalKeys; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.RoleKeys; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.launch.AbstractLauncher; +import org.apache.slider.core.launch.JavaCommandLineBuilder; +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.providers.ProviderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; + +/** + * handles the setup of the Slider AM. + * This keeps aspects of role, cluster validation and Clusterspec setup + * out of the core slider client + */ +public class SliderAMClientProvider extends AbstractClientProvider + implements SliderKeys { + + + protected static final Logger log = + LoggerFactory.getLogger(SliderAMClientProvider.class); + protected static final String NAME = "SliderAM"; + public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT + + "slideram/instance/"; + public static final String INTERNAL_JSON = + INSTANCE_RESOURCE_BASE + "internal.json"; + public static final String APPCONF_JSON = + INSTANCE_RESOURCE_BASE + "appconf.json"; + public static final String RESOURCES_JSON = + INSTANCE_RESOURCE_BASE + "resources.json"; + + public SliderAMClientProvider(Configuration conf) { + super(conf); + } + + /** + * List of roles + */ + public static final List<ProviderRole> ROLES = + new ArrayList<ProviderRole>(); + + public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX; + + public static final ProviderRole APPMASTER = + new ProviderRole(COMPONENT_AM, KEY_AM, + PlacementPolicy.EXCLUDE_FROM_FLEXING, + ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, + 0, ""); + + /** + * Initialize role list + */ + static { + ROLES.add(APPMASTER); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public List<ProviderRole> getRoles() { + return ROLES; + } + + + @Override //Client + public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, + String clustername, + Configuration configuration, + AggregateConf instanceDefinition, + Path clusterDirPath, + Path generatedConfDirPath, + boolean secure) + throws SliderException, IOException { + + super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); + //add a check for the directory being writeable by the current user + String + dataPath = instanceDefinition.getInternalOperations() + .getGlobalOptions() + .getMandatoryOption( + InternalKeys.INTERNAL_DATA_DIR_PATH); + + Path path = new Path(dataPath); + sliderFileSystem.verifyDirectoryWriteAccess(path); + Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); + sliderFileSystem.verifyDirectoryWriteAccess(historyPath); + } + + /** + * Verify that an instance definition is considered valid by the provider + * @param instanceDefinition instance definition + * @throws SliderException if the configuration is not valid + */ + public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws + SliderException { + + super.validateInstanceDefinition(instanceDefinition, fs); + + // make sure there is no negative entry in the instance count + Map<String, Map<String, String>> instanceMap = + instanceDefinition.getResources().components; + for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) { + MapOperations mapOperations = new MapOperations(entry); + int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0); + if (instances < 0) { + throw new BadClusterStateException( + "Component %s has negative instance count: %d", + mapOperations.name, + instances); + } + } + } + + /** + * The Slider AM sets up all the dependency JARs above slider.jar itself + * {@inheritDoc} + */ + public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, + Configuration serviceConf, + AbstractLauncher launcher, + AggregateConf instanceDescription, + Path snapshotConfDirPath, + Path generatedConfDirPath, + Configuration clientConfExtras, + String libdir, + Path tempPath, boolean miniClusterTestRun) + throws IOException, SliderException { + + Map<String, LocalResource> providerResources = new HashMap<>(); + + ProviderUtils.addProviderJar(providerResources, + this, + SLIDER_JAR, + fileSystem, + tempPath, + libdir, + miniClusterTestRun); + + String libDirProp = + System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + log.info("Loading all dependencies for AM."); + // If slider.tar.gz is available in hdfs use it, else upload all jars + Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); + if (fileSystem.isFile(dependencyLibTarGzip)) { + SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); + } else { + ProviderUtils.addAllDependencyJars(providerResources, + fileSystem, + tempPath, + libdir, + libDirProp); + } + addKeytabResourceIfNecessary(fileSystem, + instanceDescription, + providerResources); + + launcher.addLocalResources(providerResources); + + //also pick up all env variables from a map + launcher.copyEnvVars( + instanceDescription.getInternalOperations().getOrAddComponent( + SliderKeys.COMPONENT_AM)); + } + + /** + * If the cluster is secure, and an HDFS installed keytab is available for AM + * authentication, add this keytab as a local resource for the AM launch. + * + * @param fileSystem + * @param instanceDescription + * @param providerResources + * @throws IOException + * @throws BadConfigException if there's no keytab and it is explicitly required. + */ + protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem, + AggregateConf instanceDescription, + Map<String, LocalResource> providerResources) + throws IOException, BadConfigException { + if (UserGroupInformation.isSecurityEnabled()) { + String keytabPathOnHost = instanceDescription.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + if (SliderUtils.isUnset(keytabPathOnHost)) { + String amKeytabName = instanceDescription.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = instanceDescription.getAppConfOperations() + .getComponent(SliderKeys.COMPONENT_AM).get( + SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName, + instanceDescription.getName()); + if (fileSystem.getFileSystem().exists(keytabPath)) { + LocalResource keytabRes = fileSystem.createAmResource(keytabPath, + LocalResourceType.FILE); + + providerResources.put(SliderKeys.KEYTAB_DIR + "/" + + amKeytabName, keytabRes); + } else { + log.warn("No keytab file was found at {}.", keytabPath); + if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { + throw new BadConfigException("No keytab file was found at %s.", keytabPath); + + } else { + log.warn("The AM will be " + + "started without a kerberos authenticated identity. " + + "The application is therefore not guaranteed to remain " + + "operational beyond 24 hours."); + } + } + } + } + } + + /** + * Update the AM resource with any local needs + * @param capability capability to update + */ + public void prepareAMResourceRequirements(MapOperations sliderAM, + Resource capability) { + capability.setMemory(sliderAM.getOptionInt( + ResourceKeys.YARN_MEMORY, + capability.getMemory())); + capability.setVirtualCores( + sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); + } + + /** + * Extract any JVM options from the cluster specification and + * add them to the command line + */ + public void addJVMOptions(AggregateConf aggregateConf, + JavaCommandLineBuilder cmdLine) + throws BadConfigException { + + MapOperations sliderAM = + aggregateConf.getAppConfOperations().getMandatoryComponent( + SliderKeys.COMPONENT_AM); + cmdLine.forceIPv4().headless(); + String heap = sliderAM.getOption(RoleKeys.JVM_HEAP, + DEFAULT_JVM_HEAP); + cmdLine.setJVMHeap(heap); + String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, ""); + if (SliderUtils.isSet(jvmopts)) { + cmdLine.add(jvmopts); + } + } + + + @Override + public void prepareInstanceConfiguration(AggregateConf aggregateConf) + throws SliderException, IOException { + mergeTemplates(aggregateConf, + INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON + ); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
