http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java new file mode 100644 index 0000000..a3780cc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.services.resource; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Resource determines the amount of resources (vcores, memory, network, etc.) + * usable by a container. This field determines the resource to be applied for + * all the containers of a component or application. The resource specified at + * the app (or global) level can be overriden at the component level. Only one + * of profile OR cpu & memory are exepected. It raises a validation + * exception otherwise. + **/ + +@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.") +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +public class Resource extends BaseResource { + private static final long serialVersionUID = -6431667797380250037L; + + private String profile = null; + private Integer cpus = null; + private String memory = null; + + /** + * Each resource profile has a unique id which is associated with a + * cluster-level predefined memory, cpus, etc. + **/ + public Resource profile(String profile) { + this.profile = profile; + return this; + } + + @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.") + @JsonProperty("profile") + public String getProfile() { + return profile; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + /** + * Amount of vcores allocated to each container (optional but overrides cpus + * in profile if specified). + **/ + public Resource cpus(Integer cpus) { + this.cpus = cpus; + return this; + } + + @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).") + @JsonProperty("cpus") + public Integer getCpus() { + return cpus; + } + + public void setCpus(Integer cpus) { + this.cpus = cpus; + } + + /** + * Amount of memory allocated to each container (optional but overrides memory + * in profile if specified). Currently accepts only an integer value and + * default unit is in MB. + **/ + public Resource memory(String memory) { + this.memory = memory; + return this; + } + + @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.") + @JsonProperty("memory") + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Resource resource = (Resource) o; + return Objects.equals(this.profile, resource.profile) + && Objects.equals(this.cpus, resource.cpus) + && Objects.equals(this.memory, resource.memory); + } + + @Override + public int hashCode() { + return Objects.hash(profile, cpus, memory); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class Resource {\n"); + + sb.append(" profile: ").append(toIndentedString(profile)).append("\n"); + sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n"); + sb.append(" memory: ").append(toIndentedString(memory)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } + + @Override + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java new file mode 100644 index 0000000..4c16546 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.services.utils; + +public interface RestApiConstants { + String CONTEXT_ROOT = "/services/v1"; + String APPLICATIONS_API_RESOURCE_PATH = "/applications"; + String CONTAINERS_API_RESOURCE_PATH = "/containers"; + String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster"; + String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0"; + String METAINFO_SCHEMA_VERSION = "2.1"; + String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker"; + + String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd"; + String DEFAULT_COMPONENT_NAME = "DEFAULT"; + String DEFAULT_IMAGE = "centos:centos6"; + String DEFAULT_NETWORK = "bridge"; + String DEFAULT_COMMAND_PATH = "/usr/bin/docker"; + String DEFAULT_USE_NETWORK_SCRIPT = "yes"; + + String PLACEHOLDER_APP_NAME = "${APP_NAME}"; + String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}"; + String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}"; + + String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST"; + String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT"; + String PROPERTY_APP_LIFETIME = "docker.lifetime"; + String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER"; + Long DEFAULT_UNLIMITED_LIFETIME = -1l; + + Integer HTTP_STATUS_CODE_ACCEPTED = 202; + String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip"; + + Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200; + + String PROPERTY_PYTHON_PATH = "python.path"; + String PROPERTY_COMPONENT_TYPE = "site.global.component_type"; + String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency"; + + String COMPONENT_TYPE_EXTERNAL = "external"; + + String COMMAND_ORDER_SUFFIX_START = "-START"; + String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED"; + String EXPORT_GROUP_NAME = "QuickLinks"; + + Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001; + Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002; + Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003; + Integer ERROR_CODE_APP_NAME_INVALID = 404004; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java new file mode 100644 index 0000000..685f85a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.services.utils; + +public interface RestApiErrorMessages { + String ERROR_APPLICATION_NAME_INVALID = + "Application name is either empty or not provided"; + String ERROR_APPLICATION_NAME_INVALID_FORMAT = + "Application name is not valid - only lower case letters, digits," + + " underscore and hyphen are allowed"; + + String ERROR_APPLICATION_NOT_RUNNING = "Application not running"; + String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found"; + + String ERROR_SUFFIX_FOR_COMPONENT = + " for component %s (nor at the global level)"; + String ERROR_ARTIFACT_INVALID = "Artifact is not provided"; + String ERROR_ARTIFACT_FOR_COMP_INVALID = + ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_ARTIFACT_ID_INVALID = + "Artifact id (like docker image name) is either empty or not provided"; + String ERROR_ARTIFACT_ID_FOR_COMP_INVALID = + ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + + String ERROR_RESOURCE_INVALID = "Resource is not provided"; + String ERROR_RESOURCE_FOR_COMP_INVALID = + ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_MEMORY_INVALID = + "Application resource or memory not provided"; + String ERROR_RESOURCE_CPUS_INVALID = + "Application resource or cpus not provided"; + String ERROR_RESOURCE_CPUS_INVALID_RANGE = + "Unacceptable no of cpus specified, either zero or negative"; + String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID = + ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID = + ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE = + ERROR_RESOURCE_CPUS_INVALID_RANGE + + " for component %s (or at the global level)"; + String ERROR_CONTAINERS_COUNT_INVALID = + "Required no of containers not specified"; + String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID = + ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT; + + String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED = + "Cannot specify" + " cpus/memory along with profile"; + String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED = + ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED + + " for component %s"; + String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET = + "Resource profile is not " + "supported yet. Please specify cpus/memory."; + + String ERROR_APPLICATION_IN_USE = "Application name is already in use"; + String ERROR_NULL_ARTIFACT_ID = + "Artifact Id can not be null if artifact type is none"; + String ERROR_ABSENT_NUM_OF_INSTANCE = + "Num of instances should appear either globally or per component"; + String ERROR_ABSENT_LAUNCH_COMMAND = + "launch command should appear if type is slider-zip or none"; + + String ERROR_QUICKLINKS_FOR_COMP_INVALID = + "Quicklinks specified at component level, needs corresponding values set at application level"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java new file mode 100644 index 0000000..b1b6d7c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.services.webapp; + +import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.util.Arrays; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.mortbay.jetty.webapp.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class launches the web application using Hadoop HttpServer2 (which uses + * an embedded Jetty container). This is the entry point to your application. + * The Java command used to launch this app should call the main method. + */ +public class ApplicationApiWebApp extends AbstractService { + private static final Logger logger = LoggerFactory + .getLogger(ApplicationApiWebApp.class); + private static final String SEP = ";"; + + // REST API server for YARN native services + private HttpServer2 applicationApiServer; + + public static void main(String[] args) throws IOException { + ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp(); + try { + apiWebApp.startWebApp(); + } catch (Exception e) { + if (apiWebApp != null) { + apiWebApp.close(); + } + } + } + + public ApplicationApiWebApp() { + super(ApplicationApiWebApp.class.getName()); + } + + @Override + protected void serviceStart() throws Exception { + startWebApp(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (applicationApiServer != null) { + applicationApiServer.stop(); + } + super.serviceStop(); + } + + protected void startWebApp() throws IOException { + // The port that we should run on can be set into an environment variable + // Look for that variable and default to 9191 if it isn't there. + String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT); + if (StringUtils.isEmpty(webPort)) { + webPort = "9191"; + } + + String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST); + if (StringUtils.isEmpty(webHost)) { + webHost = InetAddress.getLocalHost().getHostName(); + } + logger.info("YARN native services REST API running on host {} and port {}", + webHost, webPort); + logger.info("Configuration = {}", getConfig()); + + applicationApiServer = new HttpServer2.Builder() + .setName("services-rest-api") + .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build(); + + String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP + + "org.apache.hadoop.yarn.services.api.impl" + SEP + + "org.apache.hadoop.yarn.services.resource" + SEP + + "org.apache.hadoop.yarn.services.utils" + SEP + + "org.apache.hadoop.yarn.services.webapp" + SEP + + GenericExceptionHandler.class.getPackage().getName() + SEP + + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT + + "/*"); + + try { + logger.info("Application starting up. Logging start..."); + applicationApiServer.start(); + logger.info("Server status = {}", applicationApiServer.toString()); + for (Configuration conf : applicationApiServer.getWebAppContext() + .getConfigurations()) { + logger.info("Configurations = {}", conf); + } + logger.info("Context Path = {}", Arrays.asList(applicationApiServer + .getWebAppContext().getContextPath())); + logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer + .getWebAppContext().getResourceBase())); + logger.info("War = {}", + Arrays.asList(applicationApiServer.getWebAppContext().getWar())); + } catch (Exception ex) { + logger.error("Hadoop HttpServer2 App **failed**", ex); + throw ex; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties new file mode 100644 index 0000000..8c679b9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for YARN Services REST API Server + +# Log rotation based on size (100KB) with a max of 10 backup files +log4j.rootLogger=INFO, restservicelog +log4j.threshhold=ALL + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n + +log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender +log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout +log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log +log4j.appender.restservicelog.MaxFileSize=1GB +log4j.appender.restservicelog.MaxBackupIndex=10 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n + +# for debugging REST API Service +#log4j.logger.org.apache.hadoop.yarn.services=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN +log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR + +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN + +log4j.logger.org.mortbay.log=DEBUG http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app new file mode 100644 index 0000000..6a077b1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING. http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh new file mode 100644 index 0000000..9f15b7e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export SLIDER_VERSION=${project.version} +export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION} +export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +export LIB_PARENT_DIR=`dirname $SCRIPT_DIR` +export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf} +export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191} +export APP_RUNAS_USER=${APP_RUNAS_USER:-root} +export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/} +export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log" +$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000..f2f8b5b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" + version="3.0"> + + <servlet> + <servlet-name>Jersey REST API</servlet-name> + <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class> + <init-param> + <param-name>com.sun.jersey.config.property.packages</param-name> + <param-value>org.apache.hadoop.yarn.services.webapp,org.apache.hadoop.yarn.services.api,org.apache.hadoop.yarn.services.resource,org.apache.hadoop.yarn.services.api.impl</param-value> + </init-param> + <init-param> + <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name> + <param-value>true</param-value> + </init-param> + <load-on-startup>1</load-on-startup> + </servlet> + <servlet-mapping> + <servlet-name>Jersey REST API</servlet-name> + <url-pattern>/*</url-pattern> + </servlet-mapping> +</web-app> http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java new file mode 100644 index 0000000..a03ab69 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.services.api.impl; + +import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; +import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.services.resource.Application; +import org.apache.hadoop.yarn.services.resource.Artifact; +import org.apache.hadoop.yarn.services.resource.Resource; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test class for application life time monitor feature test. + */ +@RunWith(PowerMockRunner.class) +@SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService") +public class TestApplicationApiService { + private static final Logger logger = LoggerFactory + .getLogger(TestApplicationApiService.class); + private static String EXCEPTION_PREFIX = "Should have thrown exception: "; + private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: "; + private ApplicationApiService appApiService; + + @Before + public void setup() throws Exception { + appApiService = new ApplicationApiService(); + } + + @After + public void tearDown() throws Exception { + } + + @Test(timeout = 90000) + public void testValidateApplicationPostPayload() throws Exception { + Application app = new Application(); + Map<String, String> compNameArtifactIdMap = new HashMap<>(); + + // no name + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no name"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); + } + + // bad format name + String[] badNames = { "4finance", "Finance", "finance@home" }; + for (String badName : badNames) { + app.setName(badName); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, + e.getMessage()); + } + } + + // no artifact + app.setName("finance_home"); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); + } + + // no artifact id + Artifact artifact = new Artifact(); + app.setArtifact(artifact); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); + } + + // if artifact is of type APPLICATION then everything is valid here + artifact.setType(Artifact.TypeEnum.APPLICATION); + artifact.setId("app.io/hbase:facebook_0.2"); + app.setNumberOfContainers(5l); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + } catch (IllegalArgumentException e) { + logger.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + // default-component, default-lifetime and the property component_type + // should get assigned here + Assert.assertEquals(app.getComponents().get(0).getName(), + DEFAULT_COMPONENT_NAME); + Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); + Assert.assertEquals("Property not set", + app.getConfiguration().getProperties().get(PROPERTY_COMPONENT_TYPE), + COMPONENT_TYPE_EXTERNAL); + + // unset artifact type, default component and no of containers to test other + // validation logic + artifact.setType(null); + app.setComponents(null); + app.setNumberOfContainers(null); + + // resource not specified + artifact.setId("docker.io/centos:centos7"); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no resource"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); + } + + // memory not specified + Resource res = new Resource(); + app.setResource(res); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no memory"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); + } + + // cpus not specified + res.setMemory("2gb"); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + "application with no cpu"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage()); + } + + // invalid no of cpus + res.setCpus(-2); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail( + EXCEPTION_PREFIX + "application with invalid no of cpups"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage()); + } + + // number of containers not specified + res.setCpus(2); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail( + EXCEPTION_PREFIX + "application with no container count"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_CONTAINERS_COUNT_INVALID, e.getMessage()); + } + + // specifying profile along with cpus/memory raises exception + res.setProfile("hbase_finance_large"); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + + "application with resource profile along with cpus/memory"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED, + e.getMessage()); + } + + // currently resource profile alone is not supported. + // TODO: remove the next test once it is supported. + res.setCpus(null); + res.setMemory(null); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + Assert.fail(EXCEPTION_PREFIX + + "application with resource profile only - NOT SUPPORTED"); + } catch (IllegalArgumentException e) { + Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET, + e.getMessage()); + } + + // unset profile here and add cpus/memory back + res.setProfile(null); + res.setCpus(2); + res.setMemory("2gb"); + + // everything valid here + app.setNumberOfContainers(5l); + try { + appApiService.validateApplicationPostPayload(app, + compNameArtifactIdMap); + } catch (IllegalArgumentException e) { + logger.error("application attributes specified should be valid here", e); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); + } + + // Now test with components + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index 493e03a..2cec8bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -37,7 +37,7 @@ <module>hadoop-yarn-applications-distributedshell</module> <module>hadoop-yarn-applications-unmanaged-am-launcher</module> <module>hadoop-yarn-slider</module> - + <module>hadoop-yarn-services-api</module> </modules> <profiles> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org