http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java new file mode 100644 index 0000000..7e8cf5b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.util; + +public interface RestApiConstants { + String CONTEXT_ROOT = "/services/v1"; + String APPLICATIONS_API_RESOURCE_PATH = "/applications"; + String CONTAINERS_API_RESOURCE_PATH = "/containers"; + String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster"; + String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";; + String METAINFO_SCHEMA_VERSION = "2.1"; + String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker"; + + String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd"; + String DEFAULT_COMPONENT_NAME = "DEFAULT"; + String DEFAULT_IMAGE = "centos:centos6"; + String DEFAULT_NETWORK = "bridge"; + String DEFAULT_COMMAND_PATH = "/usr/bin/docker"; + String DEFAULT_USE_NETWORK_SCRIPT = "yes"; + + String PLACEHOLDER_APP_NAME = "${APP_NAME}"; + String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}"; + String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}"; + + String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST"; + String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT"; + String PROPERTY_APP_LIFETIME = "docker.lifetime"; + String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER"; + Long DEFAULT_UNLIMITED_LIFETIME = -1l; + + Integer HTTP_STATUS_CODE_ACCEPTED = 202; + String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip"; + + Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200; + + String PROPERTY_PYTHON_PATH = "python.path"; + String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency"; + + String COMMAND_ORDER_SUFFIX_START = "-START"; + String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED"; + String EXPORT_GROUP_NAME = "QuickLinks"; + + Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001; + Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002; + Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003; + Integer ERROR_CODE_APP_NAME_INVALID = 404004; + +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java new file mode 100644 index 0000000..0f6247d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.util; + +public interface RestApiErrorMessages { + String ERROR_APPLICATION_NAME_INVALID = + "Application name is either empty or not provided"; + String ERROR_APPLICATION_NAME_INVALID_FORMAT = + "Application name is not valid - only lower case letters, digits," + + " underscore and hyphen are allowed"; + + String ERROR_APPLICATION_NOT_RUNNING = "Application not running"; + String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found"; + String ERROR_APPLICATION_IN_USE = "Application already exists in started" + + " state"; + String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in" + + " stopped/failed state (either restart with PUT or destroy with DELETE" + + " before creating a new one)"; + + String ERROR_SUFFIX_FOR_COMPONENT = + " for component %s (nor at the global level)"; + String ERROR_ARTIFACT_INVALID = "Artifact is not provided"; + String ERROR_ARTIFACT_FOR_COMP_INVALID = + ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_ARTIFACT_ID_INVALID = + "Artifact id (like docker image name) is either empty or not provided"; + String ERROR_ARTIFACT_ID_FOR_COMP_INVALID = + ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + + String ERROR_RESOURCE_INVALID = "Resource is not provided"; + String ERROR_RESOURCE_FOR_COMP_INVALID = + ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_MEMORY_INVALID = + "Application resource or memory not provided"; + String ERROR_RESOURCE_CPUS_INVALID = + "Application resource or cpus not provided"; + String ERROR_RESOURCE_CPUS_INVALID_RANGE = + "Unacceptable no of cpus specified, either zero or negative"; + String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID = + ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID = + ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE = + ERROR_RESOURCE_CPUS_INVALID_RANGE + + " for component %s (or at the global level)"; + String ERROR_CONTAINERS_COUNT_INVALID = + "Required no of containers not specified"; + String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID = + ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + + String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED = + "Cannot specify" + " cpus/memory along with profile"; + String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED = + ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED + + " for component %s"; + String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET = + "Resource profile is not " + "supported yet. Please specify cpus/memory."; + + String ERROR_NULL_ARTIFACT_ID = + "Artifact Id can not be null if artifact type is none"; + String ERROR_ABSENT_NUM_OF_INSTANCE = + "Num of instances should appear either globally or per component"; + String ERROR_ABSENT_LAUNCH_COMMAND = + "launch command should appear if type is slider-zip or none"; + + String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at" + + " component level, needs corresponding values set at application level"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java new file mode 100644 index 0000000..776ce00 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.util; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Configuration; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderUtils; + +public class ServiceApiUtil { + + @VisibleForTesting + public static void validateApplicationPostPayload(Application application) { + if (StringUtils.isEmpty(application.getName())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); + } + if (!SliderUtils.isClusternameValid(application.getName())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT); + } + + // If the application has no components do top-level checks + if (!hasComponent(application)) { + // artifact + if (application.getArtifact() == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_INVALID); + } + if (StringUtils.isEmpty(application.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + + // If artifact is of type APPLICATION, add a slider specific property + if (application.getArtifact().getType() + == Artifact.TypeEnum.APPLICATION) { + if (application.getConfiguration() == null) { + application.setConfiguration(new Configuration()); + } + } + // resource + validateApplicationResource(application.getResource(), null, + application.getArtifact().getType()); + + // container size + if (application.getNumberOfContainers() == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID); + } + + // Since it is a simple app with no components, create a default component + application.getComponents().add(createDefaultComponent(application)); + } else { + // If the application has components, then run checks for each component. + // Let global values take effect if component level values are not + // provided. + Artifact globalArtifact = application.getArtifact(); + Resource globalResource = application.getResource(); + Long globalNumberOfContainers = application.getNumberOfContainers(); + for (Component comp : application.getComponents()) { + // artifact + if (comp.getArtifact() == null) { + comp.setArtifact(globalArtifact); + } + // If still null raise validation exception + if (comp.getArtifact() == null) { + throw new IllegalArgumentException(String + .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID, + comp.getName())); + } + if (StringUtils.isEmpty(comp.getArtifact().getId())) { + throw new IllegalArgumentException(String + .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID, + comp.getName())); + } + + // If artifact is of type APPLICATION, add a slider specific property + if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { + if (comp.getConfiguration() == null) { + comp.setConfiguration(new Configuration()); + } + comp.setName(comp.getArtifact().getId()); + } + + // resource + if (comp.getResource() == null) { + comp.setResource(globalResource); + } + validateApplicationResource(comp.getResource(), comp, + comp.getArtifact().getType()); + + // container count + if (comp.getNumberOfContainers() == null) { + comp.setNumberOfContainers(globalNumberOfContainers); + } + if (comp.getNumberOfContainers() == null) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, + comp.getName())); + } + } + } + + // Application lifetime if not specified, is set to unlimited lifetime + if (application.getLifetime() == null) { + application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); + } + } + + private static void validateApplicationResource(Resource resource, + Component comp, Artifact.TypeEnum artifactType) { + // Only apps/components of type APPLICATION can skip resource requirement + if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { + return; + } + if (resource == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String + .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID, + comp.getName())); + } + // One and only one of profile OR cpus & memory can be specified. Specifying + // both raises validation error. + if (StringUtils.isNotEmpty(resource.getProfile()) && ( + resource.getCpus() != null || StringUtils + .isNotEmpty(resource.getMemory()))) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, + comp.getName())); + } + // Currently resource profile is not supported yet, so we will raise + // validation error if only resource profile is specified + if (StringUtils.isNotEmpty(resource.getProfile())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); + } + + String memory = resource.getMemory(); + Integer cpus = resource.getCpus(); + if (StringUtils.isEmpty(memory)) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus <= 0) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String + .format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, + comp.getName())); + } + } + + public static boolean hasComponent(Application application) { + if (application.getComponents() == null || application.getComponents() + .isEmpty()) { + return false; + } + return true; + } + + public static Component createDefaultComponent(Application app) { + Component comp = new Component(); + comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME); + comp.setArtifact(app.getArtifact()); + comp.setResource(app.getResource()); + comp.setNumberOfContainers(app.getNumberOfContainers()); + comp.setLaunchCommand(app.getLaunchCommand()); + return comp; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto index b8bdc59..bfcab23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto @@ -80,22 +80,14 @@ message UpgradeContainersRequestProto { message UpgradeContainersResponseProto { } -/** - * flex the cluster - */ -message FlexClusterRequestProto { - required string clusterSpec = 1; +message FlexComponentRequestProto { + optional string name = 1; + optional int32 numberOfContainers = 2; } - -/** - * flex the cluster - */ -message FlexClusterResponseProto { - required bool response = 1; +message FlexComponentResponseProto { } - /** * void request */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto index d68da2b..8a0faf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto @@ -61,24 +61,14 @@ service SliderClusterProtocolPB { rpc upgradeContainers(UpgradeContainersRequestProto) returns(UpgradeContainersResponseProto); - /** - * Flex the cluster. - */ - rpc flexCluster(FlexClusterRequestProto) - returns(FlexClusterResponseProto); + rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto); /** * Get the current cluster status */ rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto) returns(GetJSONClusterStatusResponseProto); - - /** - * Get the instance definition - */ - rpc getInstanceDefinition(GetInstanceDefinitionRequestProto) - returns(GetInstanceDefinitionResponseProto); - + /** * List all running nodes in a role */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml index a2517d5..96bfe0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml @@ -24,10 +24,6 @@ <value>true</value> </property> <property> - <name>slider.provider.agent</name> - <value>org.apache.slider.providers.agent.AgentProviderFactory</value> - </property> - <property> <name>slider.provider.docker</name> <value>org.apache.slider.providers.docker.DockerProviderFactory</value> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java deleted file mode 100644 index b955931..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java +++ /dev/null @@ -1,157 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncher { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - Set<String> tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - Method rolledLogsIncludeMethod = null; - Method rolledLogsExcludeMethod = null; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - appSubmissionContext = EasyMock - .createNiceMock(ApplicationSubmissionContext.class); - EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) - .andReturn(appSubmissionContext).once(); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(yarnClientApp).once(); - - try { - LogAggregationContext.class.getMethod("newInstance", String.class, - String.class, String.class, String.class); - isOldApi = false; - rolledLogsIncludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsIncludePattern"); - rolledLogsExcludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsExcludePattern"); - } catch (Exception e) { - isOldApi = true; - } - } - - /** - * These tests will probably fail when compiled against hadoop 2.7+. Please - * refer to SLIDER-810. It has been purposely not modified so that it fails - * and that someone needs to modify the code in - * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are - * provided in that method as to what needs to be done. - * - * @throws Exception - */ - @Test - public void testExtractLogAggregationContext() throws Exception { - Map<String, String> options = new HashMap<String, String>(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, - " | slider*.txt |agent.out| |"); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = "slider*.txt|agent.out"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - - } - - @Test - public void testExtractLogAggregationContextEmptyIncludePattern() - throws Exception { - Map<String, String> options = new HashMap<String, String>(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - @Test - public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern() - throws Exception { - Map<String, String> options = new HashMap<String, String>(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, ""); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = ""; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - private void assertPatterns(String expectedIncludePattern, - String expectedExcludePattern) throws Exception { - if (isOldApi) { - Assert.assertEquals(expectedIncludePattern, - appMasterLauncher.logAggregationContext.getIncludePattern()); - Assert.assertEquals(expectedExcludePattern, - appMasterLauncher.logAggregationContext.getExcludePattern()); - } else { - Assert.assertEquals(expectedIncludePattern, - (String) rolledLogsIncludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - Assert.assertEquals(expectedExcludePattern, - (String) rolledLogsExcludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java deleted file mode 100644 index a8f6b26..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncherWithAmReset { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - GetNewApplicationResponse newApp; - Set<String> tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(new YarnClientApplication(newApp, - Records.newRecord(ApplicationSubmissionContext.class))); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws - Exception { - Map<String, String> options = new HashMap<String, String>(); - final String expectedInterval = Integer.toString (120000); - options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, - expectedInterval); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - String retryIntervalWindow = Long.toString(ctx - .getAttemptFailuresValidityInterval()); - Assert.assertEquals(expectedInterval, retryIntervalWindow); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() - throws Exception { - Map<String, String> options = new HashMap<String, String>(); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); - Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, - retryIntervalWindow); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java deleted file mode 100644 index a1986cd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster; - -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.core.conf.MapOperations; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class TestServiceRecordAttributes extends Assert { - - @Test - public void testAppConfigProvidedServiceRecordAttributes() throws Exception { - Map<String, String> options = new HashMap<>(); - options.put("slider.some.arbitrary.option", "arbitrary value"); - options.put("service.record.attribute.one_attribute", "one_attribute_value"); - options.put("service.record.attribute.second_attribute", "second_attribute_value"); - MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options); - options = new HashMap<>(); - options.put("some.component.attribute", "component_attribute_value"); - options.put("service.record.attribute.component_attribute", "component_attribute_value"); - MapOperations compProps = new MapOperations("TEST_COMP", options); - - SliderAppMaster appMaster = new SliderAppMaster(); - - ServiceRecord appServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord); - - assertNull("property should not be attribute", - appServiceRecord.get("slider.some.arbitrary.option")); - assertEquals("wrong value", "one_attribute_value", - appServiceRecord.get("one_attribute")); - assertEquals("wrong value", "second_attribute_value", - appServiceRecord.get("second_attribute")); - - ServiceRecord compServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord); - - assertNull("should not be attribute", - compServiceRecord.get("some.component.attribute")); - assertEquals("wrong value", "component_attribute_value", - compServiceRecord.get("component_attribute")); - - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
