http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java deleted file mode 100644 index 4286596..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.client.rest; - -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.WebResource; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.client.ClientRegistryBinder; -import org.apache.slider.api.SliderApplicationApi; -import org.apache.slider.core.registry.info.CustomRegistryConstants; - -import java.io.IOException; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION; - -/** - * Factory for the Rest client; hides the lookup and instantiation. - * <p> - * - */ -public class RestClientFactory { - - private final ClientRegistryBinder binder; - private final Client jerseyClient; - private final String user, serviceclass, instance; - - public RestClientFactory(RegistryOperations operations, - Client jerseyClient, - String user, - String serviceclass, - String instance) { - this.jerseyClient = jerseyClient; - this.user = user; - this.serviceclass = serviceclass; - this.instance = instance; - binder = new ClientRegistryBinder(operations); - } - - /** - * Locate the AM - * @return a resource to the AM - * @throws IOException any failure to resolve to the AM - */ - private WebResource locateAppmaster() throws IOException { - String restAPI = binder.lookupExternalRestAPI(user, serviceclass, instance, - CustomRegistryConstants.AM_REST_BASE); - return jerseyClient.resource(restAPI); - } - - /** - * Locate the slider AM then instantiate a client instance against - * its Application API. - * @return the instance - * @throws IOException on any failure - */ - public SliderApplicationApi createSliderAppApiClient() throws IOException { - WebResource appmaster = locateAppmaster(); - return createSliderAppApiClient(appmaster); - } - - /** - * Create a Slider application API client instance against - * its Application API. - * @param appmaster The AM to work against. - * @return the instance - * @throws IOException on any failure - */ - public SliderApplicationApi createSliderAppApiClient(WebResource appmaster) { - WebResource appResource = appmaster.path(SLIDER_PATH_APPLICATION); - return new SliderApplicationApiRestClient(jerseyClient, appResource); - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java deleted file mode 100644 index 4c376e0..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.client.rest; - -import com.google.common.base.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.representation.Form; -import org.apache.commons.lang.StringUtils; -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.api.types.ComponentInformation; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.api.SliderApplicationApi; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.ExceptionConverter; -import org.apache.slider.core.restclient.HttpVerb; -import org.apache.slider.api.types.PingInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.core.MediaType; -import java.io.IOException; -import java.util.Map; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; - -/** - * Implementation of the {@link SliderApplicationApi} - */ -public class SliderApplicationApiRestClient extends BaseRestClient - implements SliderApplicationApi { - private static final Logger log = - LoggerFactory.getLogger(SliderApplicationApiRestClient.class); - private WebResource appResource; - - /** - * Create an instance - * @param jerseyClient jersey client for operations - * @param appResource resource of application API - */ - public SliderApplicationApiRestClient(Client jerseyClient, - WebResource appResource) { - super(jerseyClient); - this.appResource = appResource; - } - - /** - * Create an instance - * @param jerseyClient jersey client for operations - * @param appmaster URL of appmaster/proxy to AM - */ - public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) { - super(jerseyClient); - WebResource amResource = jerseyClient.resource(appmaster); - amResource.type(MediaType.APPLICATION_JSON); - this.appResource = amResource.path(SLIDER_PATH_APPLICATION); - } - - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("SliderApplicationApiRestClient{"); - sb.append("appResource=").append(appResource); - sb.append('}'); - return sb.toString(); - } - - /** - * Create a resource under the application path - * @param subpath path under application - * @return a resource under the application path - */ - public WebResource applicationResource(String subpath) { - Preconditions.checkArgument(!StringUtils.isEmpty(subpath), - "empty path"); - Preconditions.checkNotNull(appResource, "Null app resource"); - return appResource.path(subpath); - } - - /** - * Get operation against a path under the Application - * @param <T> type expected - * @param subpath path - * @param c class to instantiate - * @return instance - * @throws IOException on any problem - */ - public <T> T getApplicationResource(String subpath, Class<T> c) - throws IOException { - return appResourceOperation(HttpVerb.GET, subpath, c); - } - - /** - * Get operation against a path under the Application - * @param <T> type expected - * @param subpath path - * @param t type info - * @return instance - * @throws IOException on any problem - */ - public <T> T getApplicationResource(String subpath, GenericType<T> t) - throws IOException { - return appResourceOperation(HttpVerb.GET, subpath, t); - } - - /** - * - * @param method method to exec - * @param <T> type expected - * @param subpath path - * @param c class to instantiate - * @return instance - * @throws IOException on any problem - */ - public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c) - throws IOException { - return exec(method, applicationResource(subpath), c); - } - - - /** - * Get operation against a path under the Application - * @param <T> type expected - * @param subpath path - * @param t type info - * @return instance - * @throws IOException on any problem - */ - public <T> T appResourceOperation(HttpVerb method, String subpath, - GenericType<T> t) - throws IOException { - return exec(method, applicationResource(subpath), t); - } - - - @Override - public AggregateConf getDesiredModel() throws IOException { - return getApplicationResource(MODEL_DESIRED, AggregateConf.class); - } - - @Override - public ConfTreeOperations getDesiredAppconf() throws IOException { - ConfTree resource = - getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class); - return new ConfTreeOperations(resource); - } - - @Override - public ConfTreeOperations getDesiredResources() throws IOException { - ConfTree resource = - getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class); - return new ConfTreeOperations(resource); - } - - @Override - public AggregateConf getResolvedModel() throws IOException { - return getApplicationResource(MODEL_RESOLVED, AggregateConf.class); - } - - - @Override - public ConfTreeOperations getResolvedAppconf() throws IOException { - ConfTree resource = - getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class); - return new ConfTreeOperations(resource); - } - - @Override - public ConfTreeOperations getResolvedResources() throws IOException { - ConfTree resource = - getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class); - return new ConfTreeOperations(resource); - } - - @Override - public ConfTreeOperations getLiveResources() throws IOException { - ConfTree resource = - getApplicationResource(LIVE_RESOURCES, ConfTree.class); - return new ConfTreeOperations(resource); - } - - @Override - public Map<String, ContainerInformation> enumContainers() throws - IOException { - return getApplicationResource(LIVE_CONTAINERS, - new GenericType<Map<String, ContainerInformation>>() { - }); - } - - @Override - public ContainerInformation getContainer(String containerId) throws - IOException { - return getApplicationResource(LIVE_CONTAINERS + "/" + containerId, - ContainerInformation.class); - } - - @Override - public Map<String, ComponentInformation> enumComponents() throws - IOException { - return getApplicationResource(LIVE_COMPONENTS, - new GenericType<Map<String, ComponentInformation>>() { }); - } - - @Override - public ComponentInformation getComponent(String componentName) throws - IOException { - return getApplicationResource(LIVE_COMPONENTS + "/" + componentName, - ComponentInformation.class); - } - - @Override - public NodeInformationList getLiveNodes() throws IOException { - return getApplicationResource(LIVE_NODES, NodeInformationList.class); - } - - @Override - public NodeInformation getLiveNode(String hostname) throws IOException { - return getApplicationResource(LIVE_NODES + "/" + hostname, - NodeInformation.class); - } - - @Override - public PingInformation ping(String text) throws IOException { - return pingPost(text); - } - - /** - * Ping as a GET - * @param text text to include - * @return the response - * @throws IOException on any failure - */ - public PingInformation pingGet(String text) throws IOException { - WebResource pingResource = applicationResource(ACTION_PING); - pingResource.getUriBuilder().queryParam("body", text); - return pingResource.get(PingInformation.class); - } - - /** - * Ping as a POST - * @param text text to include - * @return the response - * @throws IOException on any failure - */ - public PingInformation pingPost(String text) throws IOException { - WebResource pingResource = applicationResource(ACTION_PING); - Form f = new Form(); - f.add("text", text); - return pingResource - .type(MediaType.APPLICATION_JSON_TYPE) - .post(PingInformation.class, f); - } - - @Override - public void stop(String text) throws IOException { - WebResource resource = applicationResource(ACTION_STOP); - resource.post(text); - } - - @Override - public ApplicationLivenessInformation getApplicationLiveness() throws IOException { - return getApplicationResource(LIVE_LIVENESS, - ApplicationLivenessInformation.class); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 2d471ed..a2d4e38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -19,17 +19,6 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParametersDelegate; -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.fs.Path; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.providers.SliderProviderFactory; - -import java.io.File; -import java.util.List; -import java.util.Map; /** * Abstract Action to build things; shares args across build and http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 5919312..5f5e611 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -237,80 +237,6 @@ public class CoreFileSystem { } /** - * Build up the path string for cluster resource install location -no - * attempt to create the directory is made - * - * @return the path for resource - */ - public Path buildClusterResourcePath(String clusterName, String component) { - Preconditions.checkNotNull(clusterName); - Path path = buildClusterDirPath(clusterName); - return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component); - } - - /** - * Build up the path string for cluster resource install location -no - * attempt to create the directory is made - * - * @return the path for resource - */ - public Path buildClusterResourcePath(String clusterName) { - Preconditions.checkNotNull(clusterName); - Path path = buildClusterDirPath(clusterName); - return new Path(path, SliderKeys.RESOURCE_DIR); - } - - /** - * Create the Slider cluster path for a named cluster and all its subdirs - * This is a directory; a mkdirs() operation is executed - * to ensure that it is there. - * - * @param clustername name of the cluster - * @return the path to the cluster directory - * @throws java.io.IOException trouble - * @throws SliderException slider-specific exceptions - */ - public Path createClusterDirectories(String clustername, Configuration conf) - throws IOException, SliderException { - - - Path clusterDirectory = buildClusterDirPath(clustername); - InstancePaths instancePaths = new InstancePaths(clusterDirectory); - createClusterDirectories(instancePaths); - return clusterDirectory; - } - - /** - * Create the Slider cluster path for a named cluster and all its subdirs - * This is a directory; a mkdirs() operation is executed - * to ensure that it is there. - * - * @param instancePaths instance paths - * @throws IOException trouble - * @throws SliderException slider-specific exceptions - */ - public void createClusterDirectories(InstancePaths instancePaths) throws - IOException, SliderException { - Path instanceDir = instancePaths.instanceDir; - - verifyDirectoryNonexistent(instanceDir); - FsPermission clusterPerms = getInstanceDirectoryPermissions(); - createWithPermissions(instanceDir, clusterPerms); - createWithPermissions(instancePaths.snapshotConfPath, clusterPerms); - createWithPermissions(instancePaths.generatedConfPath, clusterPerms); - createWithPermissions(instancePaths.historyPath, clusterPerms); - createWithPermissions(instancePaths.tmpPathAM, clusterPerms); - - // Data Directory - String dataOpts = - configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS, - SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS); - log.debug("Setting data directory permissions to {}", dataOpts); - createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts)); - - } - - /** * Create a directory with the given permissions. * * @param dir directory http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index c0ef2d4..9d7c40a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -19,7 +19,6 @@ package org.apache.slider.common.tools; import com.google.common.base.Preconditions; - import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; @@ -52,23 +51,18 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.Slider; -import org.apache.slider.api.InternalKeys; import org.apache.slider.api.RoleKeys; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.Arguments; import org.apache.slider.common.params.SliderActions; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.ErrorStrings; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.main.LauncherExitCodes; -import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.services.utility.PatternValidator; import org.apache.slider.server.services.workflow.ForkedProcessService; import org.apache.zookeeper.server.util.KerberosUtil; @@ -81,7 +75,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.FileReader; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; @@ -112,7 +105,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TimeZone; -import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; @@ -1778,61 +1770,6 @@ public final class SliderUtils { VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum()); } - /** - * Set the time for an information (human, machine) timestamp pair of fields. - * The human time is the time in millis converted via the {@link Date} class. - * @param info info fields - * @param keyHumanTime name of human time key - * @param keyMachineTime name of machine time - * @param time timestamp - */ - public static void setInfoTime(Map info, - String keyHumanTime, - String keyMachineTime, - long time) { - info.put(keyHumanTime, SliderUtils.toGMTString(time)); - info.put(keyMachineTime, Long.toString(time)); - } - - public static Path extractImagePath(CoreFileSystem fs, - MapOperations internalOptions) throws - SliderException, IOException { - Path imagePath; - String imagePathOption = - internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - String appHomeOption = - internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME); - if (!isUnset(imagePathOption)) { - if (!isUnset(appHomeOption)) { - throw new BadClusterStateException( - ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); - } - imagePath = fs.createPathThatMustExist(imagePathOption); - } else { - imagePath = null; - if (isUnset(appHomeOption)) { - throw new BadClusterStateException( - ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED); - } - } - return imagePath; - } - - /** - * trigger a JVM halt with no clean shutdown at all - * @param status status code for exit - * @param text text message - * @param delay delay in millis - * @return the timer (assuming the JVM hasn't halted yet) - * - */ - public static Timer haltAM(int status, String text, int delay) { - - Timer timer = new Timer("halt timer", false); - timer.schedule(new DelayedHalt(status, text), delay); - return timer; - } - public static String propertiesToString(Properties props) { TreeSet<String> keys = new TreeSet<>(props.stringPropertyNames()); StringBuilder builder = new StringBuilder(); @@ -2379,40 +2316,6 @@ public final class SliderUtils { } /** - * return the HDFS path where the application package has been uploaded - * manually or by using slider client (install package command) - * - * @param conf configuration - * @return - */ - public static String getApplicationDefinitionPath(ConfTreeOperations conf) - throws BadConfigException { - return getApplicationDefinitionPath(conf, null); - } - - /** - * return the HDFS path where the application package has been uploaded - * manually or by using slider client (install package command) - * - * @param conf configuration - * @param roleGroup name of component - * @return - */ - public static String getApplicationDefinitionPath(ConfTreeOperations conf, - String roleGroup) - throws BadConfigException { - String appDefPath = conf.getGlobalOptions().getMandatoryOption( - AgentKeys.APP_DEF); - if (roleGroup != null) { - MapOperations component = conf.getComponent(roleGroup); - if (component != null) { - appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath); - } - } - return appDefPath; - } - - /** * return the path to the slider-client.xml used by the current running * slider command * @@ -2487,25 +2390,6 @@ public final class SliderUtils { } /** - * return a description of whether the current user has created credential - * cache files from kerberos servers - * - * @throws IOException - * @throws BadConfigException - * @throws SecurityException - * - if a security manager exists and its checkPropertyAccess - * method doesn't allow access to the specified system property. - */ - public static String checkCredentialCacheFile() throws IOException, - BadConfigException { - String result = null; - if (!Shell.WINDOWS) { - result = Shell.execCommand("klist"); - } - return result; - } - - /** * Compare the times of two applications: most recent app comes first * Specifically: the one whose start time value is greater. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java deleted file mode 100644 index f14a07a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java +++ /dev/null @@ -1,520 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.buildutils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.CoreFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.persist.ConfPersister; -import org.apache.slider.core.persist.InstancePaths; -import org.apache.slider.core.persist.LockAcquireFailedException; -import org.apache.slider.core.persist.LockHeldAction; -import org.apache.slider.core.zk.ZKPathBuilder; -import org.apache.slider.core.zk.ZookeeperUtils; -import org.apache.slider.providers.agent.AgentKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeSet; - -import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; -import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; -import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE; -import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR; -import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR; -import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME; -import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH; -import static org.apache.slider.api.OptionKeys.INTERNAL_DATA_DIR_PATH; -import static org.apache.slider.api.OptionKeys.INTERNAL_GENERATED_CONF_PATH; -import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH; -import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS; -import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH; -import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; -import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; -import static org.apache.slider.common.SliderKeys.COMPONENT_AM; -import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; -import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_EXTERNAL_APP; -import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_KEY; -import static org.apache.slider.common.tools.SliderUtils.isClusternameValid; - -/** - * Build up the instance of a cluster. - */ -public class InstanceBuilder { - - private final String clustername; - private final Configuration conf; - private final CoreFileSystem coreFS; - private final InstancePaths instancePaths; - private AggregateConf instanceDescription; - private Map<String, Path> externalAppDefs = new HashMap<>(); - private TreeSet<Integer> priorities = new TreeSet<>(); - - private static final Logger log = - LoggerFactory.getLogger(InstanceBuilder.class); - - public InstanceBuilder(CoreFileSystem coreFileSystem, - Configuration conf, - String clustername) { - this.clustername = clustername; - this.conf = conf; - this.coreFS = coreFileSystem; - Path instanceDir = coreFileSystem.buildClusterDirPath(clustername); - instancePaths = new InstancePaths(instanceDir); - - } - - public AggregateConf getInstanceDescription() { - return instanceDescription; - } - - public InstancePaths getInstancePaths() { - return instancePaths; - } - - - @Override - public String toString() { - return "Builder working with " + clustername + " at " + - getInstanceDir(); - } - - private Path getInstanceDir() { - return instancePaths.instanceDir; - } - - /** - * Initial part of the build process - * @param instanceConf - * @param provider - */ - public void init( - String provider, - AggregateConf instanceConf) { - - - this.instanceDescription = instanceConf; - - //internal is extended - ConfTreeOperations internalOps = instanceConf.getInternalOperations(); - - Map<String, Object> md = internalOps.getConfTree().metadata; - long time = System.currentTimeMillis(); - md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time)); - md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time)); - - BuildHelper.addBuildMetadata(md, "create"); - SliderUtils.setInfoTime(md, - StatusKeys.INFO_CREATE_TIME_HUMAN, - StatusKeys.INFO_CREATE_TIME_MILLIS, - System.currentTimeMillis()); - - internalOps.set(INTERNAL_AM_TMP_DIR, - instancePaths.tmpPathAM.toUri()); - internalOps.set(INTERNAL_TMP_DIR, - instancePaths.tmpPath.toUri()); - internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH, - instancePaths.snapshotConfPath.toUri()); - internalOps.set(INTERNAL_GENERATED_CONF_PATH, - instancePaths.generatedConfPath.toUri()); - internalOps.set(INTERNAL_DATA_DIR_PATH, - instancePaths.dataPath.toUri()); - internalOps.set(INTERNAL_APPDEF_DIR_PATH, - instancePaths.appDefPath.toUri()); - internalOps.set(INTERNAL_ADDONS_DIR_PATH, - instancePaths.addonsPath.toUri()); - - - internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider); - internalOps.set(OptionKeys.APPLICATION_NAME, clustername); - - } - - /** - * Set the queue used to start the application - * @param queue - * @throws BadConfigException - */ - public void setQueue(String queue) throws BadConfigException { - if(queue != null) { - if(SliderUtils.isUnset(queue)) { - throw new BadConfigException("Queue value cannot be empty."); - } - - instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue); - } - } - - /** - * Set up the image/app home path - * @param appImage path in the DFS to the tar file - * @param appHomeDir other strategy: home dir - * @throws BadConfigException if both are found - */ - public void setImageDetailsIfAvailable( - Path appImage, - String appHomeDir) throws BadConfigException { - boolean appHomeUnset = SliderUtils.isUnset(appHomeDir); - // App home or image - if (appImage != null) { - if (!appHomeUnset) { - // both args have been set - throw new BadConfigException( - ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); - } - instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH, - appImage.toUri()); - } else { - // the alternative is app home, which now MUST be set - if (!appHomeUnset) { - instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME, - appHomeDir); - } - } - } - - - /** - * Propagate any critical principals from the current site config down to the HBase one. - */ - public void propagatePrincipals() { - String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); - if (dfsPrincipal != null) { - String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX + - SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; - instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal); - } - } - - public void propagateFilename() { - String fsDefaultName = conf.get( - CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); - instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX + - CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, - fsDefaultName - ); - - instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX + - SliderXmlConfKeys.FS_DEFAULT_NAME_CLASSIC, - fsDefaultName - ); - - } - - - public void takeSnapshotOfConfDir(Path appconfdir) throws - IOException, - BadConfigException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(appconfdir.toUri(), conf); - if (!srcFS.isDirectory(appconfdir)) { - throw new BadConfigException( - "Source Configuration directory is not valid: %s", - appconfdir.toString()); - } - // bulk copy - FsPermission clusterPerms = coreFS.getInstanceDirectoryPermissions(); - // first the original from wherever to the DFS - SliderUtils.copyDirectory(conf, appconfdir, instancePaths.snapshotConfPath, - clusterPerms); - } - - - private Set<String> getExternalComponents(ConfTreeOperations ops) - throws BadConfigException { - Set<String> externalComponents = new HashSet<>(); - if (ops.getGlobalOptions().containsKey(COMPONENT_TYPE_KEY)) { - throw new BadConfigException(COMPONENT_TYPE_KEY + " must be " + - "specified per-component, not in global"); - } - - for (Entry<String, Map<String, String>> entry : ops.getComponents() - .entrySet()) { - if (COMPONENT_AM.equals(entry.getKey())) { - continue; - } - Map<String, String> options = entry.getValue(); - if (COMPONENT_TYPE_EXTERNAL_APP.equals(options.get(COMPONENT_TYPE_KEY))) { - externalComponents.add(entry.getKey()); - } - } - return externalComponents; - } - - private void mergeExternalComponent(ConfTreeOperations ops, - ConfTreeOperations externalOps, String externalComponent, - Integer priority) throws BadConfigException { - for (String subComponent : externalOps.getComponentNames()) { - if (COMPONENT_AM.equals(subComponent)) { - continue; - } - String prefix = externalComponent + COMPONENT_SEPARATOR; - log.debug("Merging options for {} into {}", subComponent, - prefix + subComponent); - MapOperations subComponentOps = ops.getOrAddComponent( - prefix + subComponent); - if (priority == null) { - SliderUtils.mergeMaps(subComponentOps, - ops.getComponent(externalComponent).options); - subComponentOps.remove(COMPONENT_TYPE_KEY); - } - - SliderUtils.mergeMapsIgnoreDuplicateKeysAndPrefixes(subComponentOps, - externalOps.getComponent(subComponent), - SliderKeys.COMPONENT_KEYS_TO_SKIP); - - // add prefix to existing prefix - String existingPrefix = subComponentOps.get(ROLE_PREFIX); - if (existingPrefix != null) { - if (!subComponent.startsWith(existingPrefix)) { - throw new BadConfigException("Bad prefix " + existingPrefix + - " for subcomponent " + subComponent + " of " + externalComponent); - } - prefix = prefix + existingPrefix; - } - subComponentOps.set(ROLE_PREFIX, prefix); - - // adjust priority - if (priority != null) { - subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY, - Integer.toString(priority)); - priorities.add(priority); - priority++; - } - } - } - - private int getNextPriority() { - if (priorities.isEmpty()) { - return 1; - } else { - return priorities.last() + 1; - } - } - - public void resolve() - throws BadConfigException, IOException, BadClusterStateException { - ConfTreeOperations appConf = instanceDescription.getAppConfOperations(); - ConfTreeOperations resources = instanceDescription.getResourceOperations(); - - for (Entry<String, Map<String, String>> entry : resources.getComponents() - .entrySet()) { - if (COMPONENT_AM.equals(entry.getKey())) { - continue; - } - if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) { - priorities.add(Integer.parseInt(entry.getValue().get( - ResourceKeys.COMPONENT_PRIORITY))); - } - } - - Set<String> externalComponents = getExternalComponents(appConf); - if (!externalComponents.isEmpty()) { - log.info("Found external components {}", externalComponents); - } - - for (String component : externalComponents) { - if (!isClusternameValid(component)) { - throw new BadConfigException(component + " is not a valid external " + - "component"); - } - Path componentClusterDir = coreFS.buildClusterDirPath(component); - try { - coreFS.verifyPathExists(componentClusterDir); - } catch (IOException e) { - throw new BadConfigException("external component " + component + - " doesn't exist"); - } - AggregateConf componentConf = new AggregateConf(); - ConfPersister persister = new ConfPersister(coreFS, componentClusterDir); - try { - persister.load(componentConf); - } catch (Exception e) { - throw new BadConfigException("Couldn't read configuration for " + - "external component " + component); - } - - ConfTreeOperations componentAppConf = componentConf.getAppConfOperations(); - String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF); - if (SliderUtils.isSet(externalAppDef)) { - Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), - component + "_" + SliderKeys.DEFAULT_APP_PKG); - componentAppConf.set(AgentKeys.APP_DEF, newAppDef); - componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef); - log.info("Copying external appdef {} to {} for {}", externalAppDef, - newAppDef, component); - externalAppDefs.put(externalAppDef, newAppDef); - externalAppDef = newAppDef.toString(); - } - - for (String rcomp : componentConf.getResourceOperations() - .getComponentNames()) { - if (COMPONENT_AM.equals(rcomp)) { - continue; - } - log.debug("Adding component {} to appConf for {}", rcomp, component); - componentAppConf.getOrAddComponent(rcomp); - } - componentConf.resolve(); - - for (String rcomp : componentConf.getResourceOperations() - .getComponentNames()) { - if (COMPONENT_AM.equals(rcomp)) { - continue; - } - String componentAppDef = componentAppConf.getComponentOpt( - rcomp, AgentKeys.APP_DEF, null); - if (SliderUtils.isUnset(componentAppDef) || - componentAppDef.equals(externalAppDef)) { - continue; - } - if (externalAppDefs.containsKey(componentAppDef)) { - log.info("Using external appdef {} for {}", - externalAppDefs.get(componentAppDef), rcomp); - } else { - String existingPrefix = componentAppConf.getComponentOpt(rcomp, - ROLE_PREFIX, null); - if (SliderUtils.isUnset(existingPrefix)) { - existingPrefix = ""; - } else { - existingPrefix = COMPONENT_SEPARATOR + SliderUtils.trimPrefix( - existingPrefix); - } - Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername), - component + existingPrefix + "_" + SliderKeys.DEFAULT_APP_PKG); - externalAppDefs.put(componentAppDef, newAppDef); - log.info("Copying external appdef {} to {} for {}", componentAppDef, - newAppDef, component + COMPONENT_SEPARATOR + rcomp); - } - componentAppConf.setComponentOpt(rcomp, AgentKeys.APP_DEF, - externalAppDefs.get(componentAppDef).toString()); - componentAppConf.appendComponentOpt(rcomp, - AgentKeys.APP_DEF_ORIGINAL, componentAppDef); - } - Set<Path> newAppDefs = new HashSet<>(); - newAppDefs.addAll(externalAppDefs.values()); - if (newAppDefs.size() != externalAppDefs.size()) { - throw new IllegalStateException("Values repeat in external appdefs " - + externalAppDefs); - } - log.info("External appdefs after {}: {}", component, externalAppDefs); - - SliderUtils.mergeMapsIgnoreDuplicateKeys( - appConf.getConfTree().credentials, - componentAppConf.getConfTree().credentials); - - mergeExternalComponent(appConf, componentAppConf, component, null); - mergeExternalComponent(resources, componentConf.getResourceOperations(), - component, getNextPriority()); - } - } - - - /** - * Persist this - * @param appconfdir conf dir - * @param overwrite if true, we don't need to create cluster dir - * @throws IOException - * @throws SliderException - * @throws LockAcquireFailedException - */ - public void persist(Path appconfdir, boolean overwrite) throws - IOException, - SliderException, - LockAcquireFailedException { - if (!overwrite) { - coreFS.createClusterDirectories(instancePaths); - } - ConfPersister persister = - new ConfPersister(coreFS, getInstanceDir()); - ConfDirSnapshotAction action = null; - if (appconfdir != null) { - action = new ConfDirSnapshotAction(appconfdir); - } - persister.save(instanceDescription, action); - for (Entry<String, Path> appDef : externalAppDefs.entrySet()) { - SliderUtils.copy(conf, new Path(appDef.getKey()), appDef.getValue()); - } - } - - /** - * Add the ZK paths to the application options. - * - * @param zkBinding ZK binding - */ - public void addZKBinding(ZKPathBuilder zkBinding) throws BadConfigException { - - String quorum = zkBinding.getAppQuorum(); - if (SliderUtils.isSet(quorum)) { - MapOperations globalAppOptions = - instanceDescription.getAppConfOperations().getGlobalOptions(); - globalAppOptions.put(ZOOKEEPER_PATH, zkBinding.getAppPath()); - globalAppOptions.put(ZOOKEEPER_QUORUM, quorum); - globalAppOptions.put(ZOOKEEPER_HOSTS, - ZookeeperUtils.convertToHostsOnlyList(quorum)); - } - } - - /** - * Class to execute the snapshotting of the configuration directory - * while the persistence lock is held. - * - * This guarantees that there won't be an attempt to launch a cluster - * until the snapshot is complete -as the write lock won't be released - * until afterwards. - */ - private class ConfDirSnapshotAction implements LockHeldAction { - - private final Path appconfdir; - - private ConfDirSnapshotAction(Path appconfdir) { - this.appconfdir = appconfdir; - } - - @Override - public void execute() throws IOException, SliderException { - - takeSnapshotOfConfDir(appconfdir); - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java deleted file mode 100644 index 262b6ee..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.buildutils; - -import org.apache.hadoop.fs.Path; -import org.apache.slider.common.tools.CoreFileSystem; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.persist.ConfPersister; -import org.apache.slider.core.persist.LockAcquireFailedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -public class InstanceIO { - protected static final Logger log = - LoggerFactory.getLogger(InstanceIO.class); - - /** - * Load in an instance definition -but do not resolve it - * @param sliderFileSystem filesystem - * @param clusterDirectory CD - * @return the unresolved aggregate configuration - * @throws IOException - * @throws SliderException - * @throws BadClusterStateException if a lock could not be acquired - */ - public static AggregateConf loadInstanceDefinitionUnresolved( - CoreFileSystem sliderFileSystem, - Path clusterDirectory) - throws IOException, SliderException { - AggregateConf instanceDefinition = new AggregateConf(); - ConfPersister persister = - new ConfPersister(sliderFileSystem, clusterDirectory); - try { - persister.load(instanceDefinition); - } catch (LockAcquireFailedException e) { - log.debug("Lock acquisition failure of {}", clusterDirectory, e); - - throw new BadClusterStateException( - "Application at %s is locked for reading", - clusterDirectory.toString()); - } - return instanceDefinition; - } - - - /** - * Update a persisted instance definition - * @param coreFS filesystem - * @param dir directory to load from - * @param instanceDefinition instance definition to save do - * @throws SliderException - * @throws IOException - * @throws LockAcquireFailedException - */ - public static void saveInstanceDefinition(CoreFileSystem coreFS, - Path dir, - AggregateConf instanceDefinition) - throws SliderException, IOException, LockAcquireFailedException { - ConfPersister persister = - new ConfPersister(coreFS, dir); - persister.save(instanceDefinition, null); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java deleted file mode 100644 index 336b4dc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.core.conf; - -import org.apache.slider.core.exceptions.BadConfigException; - -/** - * - */ -public abstract class AbstractInputPropertiesValidator implements InputPropertiesValidator{ - - abstract void validatePropertyNamePrefix(String key) throws BadConfigException; - - public void validate(ConfTreeOperations props) - throws BadConfigException { - validateGlobalProperties(props); - validateComponentProperties(props); - - } - - protected void validateComponentProperties(ConfTreeOperations props) - throws BadConfigException { - for (String compName : props.getComponentNames()) { - MapOperations mo = props.getComponent(compName); - if (mo == null) continue; - for (String key : mo.keySet()) { - validatePropertyNamePrefix(key); - } - } - } - - abstract void validateGlobalProperties(ConfTreeOperations props) - throws BadConfigException; - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java deleted file mode 100644 index a272420..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.conf; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.core.exceptions.BadConfigException; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.io.IOException; - -/** - * Aggregate Configuration. - * - * It is serializable to JSON - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public final class AggregateConf { - - private String name; - private ConfTree resources; - private ConfTree internal; - private ConfTree appConf; - - private ConfTreeOperations resourceOperations; - private ConfTreeOperations appConfOperations; - private ConfTreeOperations internalOperations; - - private String passphrase; - - public AggregateConf() { - this(new ConfTree(), new ConfTree(), new ConfTree()); - } - - public AggregateConf(String name) { - this(new ConfTree(), new ConfTree(), new ConfTree()); - this.name = name; - } - - public AggregateConf(ConfTree resources, - ConfTree appConf, - ConfTree internal) { - setResources(resources); - setAppConf(appConf); - setInternal(internal); - } - - /** - * Take a snapshot of the configuration - * @param instanceDefinition source - * @throws IOException marshalling/copying problems - */ - public AggregateConf(AggregateConf instanceDefinition) throws IOException { - ConfTreeOperations resourcesSnapshot = - ConfTreeOperations.fromInstance(instanceDefinition.getResources()); - ConfTreeOperations appConfSnapshot = - ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); - ConfTreeOperations internalsSnapshot = - ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); - //build a new aggregate from the snapshots - setResources(resourcesSnapshot.confTree); - setAppConf(appConfSnapshot.confTree); - setInternal(internalsSnapshot.confTree); - } - - public void setResources(ConfTree resources) { - this.resources = resources; - resourceOperations = new ConfTreeOperations(resources); - } - - public void setAppConf(ConfTree appConf) { - this.appConf = appConf; - appConfOperations = new ConfTreeOperations(appConf); - } - - public ConfTree getInternal() { - return internal; - } - - public void setInternal(ConfTree internal) { - this.internal = internal; - internalOperations = new ConfTreeOperations(internal); - } - - public ConfTree getResources() { - return resources; - } - - public ConfTree getAppConf() { - return appConf; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - @JsonIgnore - public ConfTreeOperations getResourceOperations() { - return resourceOperations; - } - - - @JsonIgnore - public ConfTreeOperations getAppConfOperations() { - return appConfOperations; - } - - @JsonIgnore - public ConfTreeOperations getInternalOperations() { - return internalOperations; - } - - /** - * predicate to query if all sections have data structures - * @return true if every section is non-null - */ - @JsonIgnore - public boolean isComplete() { - return resources != null && appConf != null && internal != null; - } - - public void validate() throws BadConfigException { - if (!isComplete()) { - throw new BadConfigException("Incomplete instance %s", this); - } - resourceOperations.validate(); - internalOperations.validate(); - appConfOperations.validate(); - } - - public void resolve() throws BadConfigException { - validate(); - resourceOperations.resolve(); - internalOperations.resolve(); - appConfOperations.resolve(); - } - - @JsonIgnore - public String getPassphrase() { - if (passphrase == null) { - passphrase = RandomStringUtils.randomAlphanumeric( - Integer.parseInt(SliderKeys.PASS_LEN)); - } - - return passphrase; - } - - /** - * Is this app package versioned? - * - * @return true if {@link SliderKeys#APP_VERSION} was set in the app config - * provided during creation of this app - * @since 0.80.0-incubating - */ - public boolean isVersioned() { - return StringUtils.isNotEmpty(getAppConfOperations().getGlobalOptions() - .get(SliderKeys.APP_VERSION)); - } - - /** - * string operation includes all the inner conftrees - * @return a string description - */ - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"internal\": ").append(internal); - sb.append(",\n\"resources\": ").append(resources); - sb.append(",\n\"appConf\" :").append(appConf); - sb.append('}'); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java deleted file mode 100644 index 74b6abb..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.conf; - -import org.apache.slider.core.persist.ConfTreeSerDeser; -import org.apache.slider.core.persist.PersistKeys; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * A conf tree represents one of the configuration trees - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public final class ConfTree { - - /** - * Size of an initial map. This is kept low so the cost of having - * many conf trees in a process is low. - */ - public static final int INITAL_MAP_CAPACITY = 3; - - protected static final Logger - log = LoggerFactory.getLogger(ConfTree.class); - - /** - * version counter - */ - public String schema = PersistKeys.SCHEMA; - - /** - * Metadata - */ - public Map<String, Object> metadata = new HashMap<>(INITAL_MAP_CAPACITY); - - - /** - * Global options - */ - public Map<String, String> global = - new HashMap<>(INITAL_MAP_CAPACITY); - - - /** - * Credentials - */ - public Map<String, List<String>> credentials = - new HashMap<>(INITAL_MAP_CAPACITY); - - /** - * Role options, - * role -> option -> value - */ - public Map<String, Map<String, String>> components = - new HashMap<>(INITAL_MAP_CAPACITY); - - @Override - public String toString() { - try { - return toJson(); - } catch (Exception e) { - log.warn("Failed to convert to JSON ", e); - return super.toString(); - } - } - - /** - * Convert to a JSON string - * @return a JSON string description - * @throws IOException Problems mapping/writing the object - */ - public String toJson() throws IOException { - return ConfTreeSerDeser.toString(this); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java deleted file mode 100644 index 526e17d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ /dev/null @@ -1,527 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.conf; - -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.persist.ConfTreeSerDeser; -import org.apache.slider.core.persist.PersistKeys; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.map.JsonMappingException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class ConfTreeOperations { - - public final ConfTree confTree; - private final MapOperations globalOptions; - - protected static final Logger - log = LoggerFactory.getLogger(ConfTreeOperations.class); - - - public ConfTreeOperations(ConfTree confTree) { - assert confTree != null : "null tree"; - assert confTree.components != null : "null tree components"; - this.confTree = confTree; - globalOptions = new MapOperations("global", confTree.global); - } - - /** - * Get the underlying conf tree - * @return the tree - */ - public ConfTree getConfTree() { - return confTree; - } - - /** - * Validate the configuration - * @throws BadConfigException - */ - public void validate() throws BadConfigException { - validate(null); - } - - /** - * Validate the configuration - * @param validator a provided properties validator - * @throws BadConfigException - */ - public void validate(InputPropertiesValidator validator) throws BadConfigException { - String version = confTree.schema; - if (version == null) { - throw new BadConfigException("'version' undefined"); - } - if (!PersistKeys.SCHEMA.equals(version)) { - throw new BadConfigException( - "version %s incompatible with supported version %s", - version, - PersistKeys.SCHEMA); - } - if (validator != null) { - validator.validate(this); - } - } - - /** - * Resolve a ConfTree by mapping all global options into each component - * -if there is none there already - */ - public void resolve() { - for (Map.Entry<String, Map<String, String>> comp : confTree.components.entrySet()) { - mergeInGlobal(comp.getValue()); - } - } - - /** - * Merge any options - * @param component dest values - */ - public void mergeInGlobal(Map<String, String> component) { - SliderUtils.mergeMapsIgnoreDuplicateKeys(component, confTree.global); - } - - /** - * Get operations on the global set - * @return a wrapped map - */ - public MapOperations getGlobalOptions() { - return globalOptions; - } - - - /** - * look up a component and return its options - * @param component component name - * @return component mapping or null - */ - public MapOperations getComponent(String component) { - Map<String, String> instance = confTree.components.get(component); - if (instance != null) { - return new MapOperations(component, instance); - } - return null; - } - - /** - * look up a component and return its options with the specified replacements - * @param component component name - * @param replacementOptions replacement options - * @return component mapping or null - */ - public MapOperations getComponent(String component, Map<String,String> - replacementOptions) { - Map<String, String> instance = confTree.components.get(component); - if (instance != null) { - Map<String, String> newInstance = new HashMap<>(); - newInstance.putAll(instance); - newInstance.putAll(replacementOptions); - return new MapOperations(component, newInstance); - } - return null; - } - - /** - * Get at the underlying component map - * @return a map of components. This is the raw ConfTree data structure - */ - public Map<String, Map<String, String>> getComponents() { - return confTree.components; - } - - /** - * Get a component -adding it to the components map if - * none with that name exists - * @param name role - * @return role mapping - */ - public MapOperations getOrAddComponent(String name) { - MapOperations operations = getComponent(name); - if (operations != null) { - return operations; - } - //create a new instances - Map<String, String> map = new HashMap<>(); - confTree.components.put(name, map); - return new MapOperations(name, map); - } - - - /* - * return the Set of names names - */ - @JsonIgnore - public Set<String> getComponentNames() { - return new HashSet<String>(confTree.components.keySet()); - } - - - - /** - * Get a component whose presence is mandatory - * @param name component name - * @return the mapping - * @throws BadConfigException if the name is not there - */ - public MapOperations getMandatoryComponent(String name) throws - BadConfigException { - MapOperations ops = getComponent(name); - if (ops == null) { - throw new BadConfigException("Missing component " + name); - } - return ops; - } - - /** - * Set a global option, converting it to a string as needed - * @param key key - * @param value non null value - */ - public void set(String key, Object value) { - globalOptions.put(key, value.toString()); - } - /** - * get a global option - * @param key key - * @return value or null - * - */ - public String get(String key) { - return globalOptions.get(key); - } - /** - * append to a global option - * @param key key - * @return value - * - */ - public String append(String key, String value) { - if (SliderUtils.isUnset(value)) { - return null; - } - if (globalOptions.containsKey(key)) { - globalOptions.put(key, globalOptions.get(key) + "," + value); - } else { - globalOptions.put(key, value); - } - return globalOptions.get(key); - } - - /** - * Propagate all global keys matching a prefix - * @param src source - * @param prefix prefix - */ - public void propagateGlobalKeys(ConfTree src, String prefix) { - Map<String, String> global = src.global; - for (Map.Entry<String, String> entry : global.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - set(key, entry.getValue()); - } - } - } - - /** - * Propagate all global keys matching a prefix - * @param src source - * @param prefix prefix - */ - public void propagateGlobalKeys(ConfTreeOperations src, String prefix) { - propagateGlobalKeys(src.confTree, prefix); - } - - /** - * Merge the map of a single component - * @param component component name - * @param map map to merge - */ - public void mergeSingleComponentMap(String component, Map<String, String> map) { - MapOperations comp = getOrAddComponent(component); - comp.putAll(map); - } - /** - * Merge the map of a single component - * @param component component name - * @param map map to merge - */ - public void mergeSingleComponentMapPrefix(String component, - Map<String, String> map, - String prefix, - boolean overwrite) { - boolean needsMerge = false; - for (Map.Entry<String, String> entry : map.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - needsMerge = true; - break; - } - } - if (!needsMerge) { - return; - } - MapOperations comp = getOrAddComponent(component); - comp.mergeMapPrefixedKeys(map,prefix, overwrite); - } - - /** - * Merge in components - * @param commandOptions component options on the CLI - */ - public void mergeComponents(Map<String, Map<String, String>> commandOptions) { - for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) { - mergeSingleComponentMap(entry.getKey(), entry.getValue()); - } - } - - /** - * Merge in components - * @param commandOptions component options on the CLI - */ - public void mergeComponentsPrefix(Map<String, - Map<String, String>> commandOptions, - String prefix, - boolean overwrite) { - for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) { - mergeSingleComponentMapPrefix(entry.getKey(), entry.getValue(), prefix, overwrite); - } - } - - /** - * Merge in another tree -no overwrites of global or conf data - * (note that metadata does a naive putAll merge/overwrite) - * @param that the other tree - */ - public void mergeWithoutOverwrite(ConfTree that) { - - getGlobalOptions().mergeWithoutOverwrite(that.global); - confTree.metadata.putAll(that.metadata); - confTree.credentials.putAll(that.credentials); - - for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) { - MapOperations comp = getOrAddComponent(entry.getKey()); - comp.mergeWithoutOverwrite(entry.getValue()); - } - } - - /** - * Merge in another tree with overwrites - * @param that the other tree - */ - public void merge(ConfTree that) { - - getGlobalOptions().putAll(that.global); - confTree.metadata.putAll(that.metadata); - confTree.credentials.putAll(that.credentials); - - for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) { - MapOperations comp = getOrAddComponent(entry.getKey()); - comp.putAll(entry.getValue()); - } - } - - - /** - * Load from a resource. The inner conf tree is the loaded data -unresolved - * @param resource resource - * @return loaded value - * @throws IOException load failure - */ - public static ConfTreeOperations fromResource(String resource) throws - IOException { - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTreeOperations ops = new ConfTreeOperations( - confTreeSerDeser.fromResource(resource) ); - return ops; - } - - /** - * Load from a resource. The inner conf tree is the loaded data -unresolved - * @param resource resource - * @return loaded value - * @throws IOException load failure - */ - public static ConfTreeOperations fromFile(File resource) throws - IOException { - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTreeOperations ops = new ConfTreeOperations( - confTreeSerDeser.fromFile(resource) ); - return ops; - } - - /** - * Build from an existing instance -which is cloned via JSON ser/deser - * @param instance the source instance - * @return loaded value - * @throws IOException load failure - */ - public static ConfTreeOperations fromInstance(ConfTree instance) throws - IOException { - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTreeOperations ops = new ConfTreeOperations( - confTreeSerDeser.fromJson(confTreeSerDeser.toJson(instance)) ); - return ops; - } - - /** - * Load from a file and merge it in - * @param file file - * @throws IOException any IO problem - * @throws BadConfigException if the file is invalid - */ - public void mergeFile(File file) throws IOException, BadConfigException { - mergeFile(file, null); - } - - /** - * Load from a file and merge it in - * @param file file - * @param validator properties validator - * @throws IOException any IO problem - * @throws BadConfigException if the file is invalid - */ - public void mergeFile(File file, InputPropertiesValidator validator) throws IOException, BadConfigException { - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTree tree = confTreeSerDeser.fromFile(file); - ConfTreeOperations ops = new ConfTreeOperations(tree); - ops.validate(validator); - merge(ops.confTree); - } - - @Override - public String toString() { - return confTree.toString(); - } - - /** - * Convert to a JSON string - * @return a JSON string description - */ - public String toJson() throws IOException, - JsonGenerationException, - JsonMappingException { - return confTree.toJson(); - } - - /** - * Get a component option - * @param name component name - * @param option option name - * @param defVal default value - * @return resolved value - */ - public String getComponentOpt(String name, String option, String defVal) { - MapOperations roleopts = getComponent(name); - if (roleopts == null) { - return defVal; - } - return roleopts.getOption(option, defVal); - } - - /** - * Get a component opt; use {@link Integer#decode(String)} so as to take hex - * oct and bin values too. - * - * @param name component name - * @param option option name - * @param defVal default value - * @return parsed value - * @throws NumberFormatException if the role could not be parsed. - */ - public int getComponentOptInt(String name, String option, int defVal) { - String val = getComponentOpt(name, option, Integer.toString(defVal)); - return Integer.decode(val); - } - - /** - * Get a component opt as a boolean using {@link Boolean#valueOf(String)}. - * - * @param name component name - * @param option option name - * @param defVal default value - * @return parsed value - * @throws NumberFormatException if the role could not be parsed. - */ - public boolean getComponentOptBool(String name, String option, boolean defVal) { - String val = getComponentOpt(name, option, Boolean.toString(defVal)); - return Boolean.valueOf(val); - } - - /** - * Set a component option, creating the component if necessary - * @param component component name - * @param option option name - * @param val value - */ - public void setComponentOpt(String component, String option, String val) { - Map<String, String> roleopts = getOrAddComponent(component); - roleopts.put(option, val); - } - - /** - * Set an integer role option, creating the role if necessary - * @param role role name - * @param option option name - * @param val integer value - */ - public void setComponentOpt(String role, String option, int val) { - setComponentOpt(role, option, Integer.toString(val)); - } - /** - * Set a long role option, creating the role if necessary - * @param role role name - * @param option option name - * @param val long value - */ - public void setComponentOpt(String role, String option, long val) { - setComponentOpt(role, option, Long.toString(val)); - } - - /** - * append to a component option - * @param key key - * @return value - * - */ - public String appendComponentOpt(String role, String key, String value) { - if (SliderUtils.isUnset(value)) { - return null; - } - MapOperations roleopts = getComponent(role); - if (roleopts == null) { - return null; - } - - if (roleopts.containsKey(key)) { - roleopts.put(key, roleopts.get(key) + "," + value); - } else { - roleopts.put(key, value); - } - return roleopts.get(key); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java deleted file mode 100644 index 237c240..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.core.conf; - -import org.apache.slider.core.exceptions.BadConfigException; - -/** - * - */ -public interface InputPropertiesValidator { - void validate(ConfTreeOperations props) throws BadConfigException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java deleted file mode 100644 index 19f6f8d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.core.conf; - -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.core.exceptions.BadConfigException; - -/** - * - */ -public class ResourcesInputPropertiesValidator - extends AbstractInputPropertiesValidator { - - void validatePropertyNamePrefix(String key) throws BadConfigException { - if (!key.startsWith("yarn.") && !key.equals(ResourceKeys.UNIQUE_NAMES)) { - throw new BadConfigException( - "argument %s does not have 'yarn.' prefix", key); - } - } - - protected void validateGlobalProperties(ConfTreeOperations props) - throws BadConfigException { - for (String key : props.getGlobalOptions().keySet()) { - validatePropertyNamePrefix(key); - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
