YARN-7423. Improve service client loggings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0 Branch: refs/heads/trunk Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b Parents: a6c4bd7 Author: Billie Rinaldi <bil...@apache.org> Authored: Sat Nov 4 12:27:42 2017 -0700 Committer: Jian He <jia...@apache.org> Committed: Mon Nov 6 13:30:20 2017 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/service/ServiceMaster.java | 4 +- .../yarn/service/api/records/Configuration.java | 8 +- .../yarn/service/client/ServiceClient.java | 50 +- .../yarn/service/component/Component.java | 10 +- .../component/instance/ComponentInstance.java | 4 +- .../containerlaunch/AbstractLauncher.java | 6 +- .../containerlaunch/ClasspathConstructor.java | 4 +- .../containerlaunch/CommandLineBuilder.java | 4 +- .../containerlaunch/JavaCommandLineBuilder.java | 4 +- .../yarn/service/monitor/probe/HttpProbe.java | 4 +- .../yarn/service/monitor/probe/PortProbe.java | 4 +- .../provider/AbstractClientProvider.java | 8 +- .../provider/AbstractProviderService.java | 4 +- .../yarn/service/provider/ProviderUtils.java | 10 +- .../registry/YarnRegistryViewForProviders.java | 6 +- .../yarn/service/utils/CoreFileSystem.java | 2 +- .../hadoop/yarn/service/utils/PortScanner.java | 4 +- .../yarn/service/utils/ServiceApiUtil.java | 2 +- .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++++++++++++++ .../hadoop/yarn/service/utils/SliderUtils.java | 564 ------------------- .../yarn/service/utils/ZookeeperUtils.java | 4 +- .../service/conf/TestValidateServiceNames.java | 1 - 22 files changed, 614 insertions(+), 638 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 2d86a5a..b0b4f06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService { } protected ContainerId getAMContainerId() throws BadClusterStateException { - return ContainerId.fromString(SliderUtils.mandatoryEnvVariable( + return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable( ApplicationConstants.Environment.CONTAINER_ID.name())); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java index dc6dff8..859c1ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java @@ -24,7 +24,7 @@ import io.swagger.annotations.ApiModelProperty; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import java.io.Serializable; import java.util.ArrayList; @@ -201,9 +201,9 @@ public class Configuration implements Serializable { * this ConfigFile. */ public synchronized void mergeFrom(Configuration that) { - SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that + ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that .getProperties()); - SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv()); + ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv()); Map<String, ConfigFile> thatMap = new HashMap<>(); for (ConfigFile file : that.getFiles()) { @@ -212,7 +212,7 @@ public class Configuration implements Serializable { for (ConfigFile thisFile : files) { if(thatMap.containsKey(thisFile.getDestFile())) { ConfigFile thatFile = thatMap.get(thisFile.getDestFile()); - SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(), + ServiceUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(), thatFile.getProperties()); thatMap.remove(thisFile.getDestFile()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 775a642..11cd30d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -69,7 +69,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.utils.ZookeeperUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; @@ -80,19 +80,13 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.text.MessageFormat; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*; import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser; -import static org.apache.hadoop.yarn.service.utils.SliderUtils.*; +import static org.apache.hadoop.yarn.service.utils.ServiceUtils.*; @InterfaceAudience.Public @InterfaceStability.Unstable @@ -169,7 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, fileName); } Path filePath = new Path(file.getAbsolutePath()); - LOG.info("Loading service definition from: " + filePath); + LOG.info("Loading service definition from local FS: " + filePath); Service service = jsonSerDeser .load(FileSystem.getLocal(getConfig()), filePath); if (!StringUtils.isEmpty(serviceName)) { @@ -482,7 +476,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM); // though if neither is set: trouble - if (SliderUtils.isUnset(registryQuorum)) { + if (ServiceUtils.isUnset(registryQuorum)) { throw new BadConfigException( "No Zookeeper quorum provided in the" + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM); @@ -505,7 +499,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, types.add(YarnServiceConstants.APP_TYPE); Set<String> tags = null; if (serviceName != null) { - tags = Collections.singleton(SliderUtils.createNameTag(serviceName)); + tags = Collections.singleton(ServiceUtils.createNameTag(serviceName)); } GetApplicationsRequest request = GetApplicationsRequest.newInstance(); request.setApplicationTypes(types); @@ -652,7 +646,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, // write out the path output CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = CLI.build(); - LOG.info("AM launch command: {}", cmdStr); + LOG.debug("AM launch command: {}", cmdStr); return cmdStr; } @@ -671,12 +665,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, } if (!UserGroupInformation.isSecurityEnabled()) { String userName = UserGroupInformation.getCurrentUser().getUserName(); - LOG.info("Run as user " + userName); + LOG.debug("Run as user " + userName); // HADOOP_USER_NAME env is used by UserGroupInformation when log in // This env makes AM run as this user env.put("HADOOP_USER_NAME", userName); } - LOG.info("AM env: \n{}", stringifyMap(env)); + LOG.debug("AM env: \n{}", stringifyMap(env)); return env; } @@ -689,11 +683,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); if (fs.isFile(dependencyLibTarGzip)) { - LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/" + LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/" + dependencyLibTarGzip); - SliderUtils.putAmTarGzipAndUpdate(localResources, fs); + fs.submitTarGzipAndUpdate(localResources); } else { - String[] libs = SliderUtils.getLibDirs(); + String[] libs = ServiceUtils.getLibDirs(); + LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + + " apps, pre-upload dependency jars to HDFS " + + "using command: yarn app -enableFastLaunch"); for (String libDirProp : libs) { ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib", libDirProp); @@ -740,7 +737,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, ApplicationId appId = submitApp(service); service.setId(appId.toString()); // write app definition on to hdfs - createDirAndPersistApp(appDir, service); + Path appJson = persistAppDef(appDir, service); + LOG.info("Persisted service " + service.getName() + " at " + appJson); return 0; } @@ -763,16 +761,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, throws IOException, SliderException { FsPermission appDirPermission = new FsPermission("750"); fs.createWithPermissions(appDir, appDirPermission); - persistAppDef(appDir, service); + Path appJson = persistAppDef(appDir, service); + LOG.info("Persisted service " + service.getName() + " at " + appJson); } - private void persistAppDef(Path appDir, Service service) - throws IOException { + private Path persistAppDef(Path appDir, Service service) throws IOException { Path appJson = new Path(appDir, service.getName() + ".json"); - jsonSerDeser - .save(fs.getFileSystem(), appJson, service, true); - LOG.info( - "Persisted service " + service.getName() + " at " + appJson); + jsonSerDeser.save(fs.getFileSystem(), appJson, service, true); + return appJson; } private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, @@ -922,7 +918,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return EXIT_SUCCESS; } - String[] libDirs = SliderUtils.getLibDirs(); + String[] libDirs = ServiceUtils.getLibDirs(); if (libDirs.length > 0) { File tempLibTarGzipFile = File.createTempFile( YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_", http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index 7667a53..7208f39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Apps; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils; import org.apache.hadoop.yarn.service.monitor.probe.Probe; import org.slf4j.Logger; @@ -419,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> { public boolean areDependenciesReady() { List<String> dependencies = componentSpec.getDependencies(); - if (SliderUtils.isEmpty(dependencies)) { + if (ServiceUtils.isEmpty(dependencies)) { return true; } for (String dependency : dependencies) { @@ -445,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> { public Map<String, String> getDependencyHostIpTokens() { Map<String, String> tokens = new HashMap<>(); List<String> dependencies = componentSpec.getDependencies(); - if (SliderUtils.isEmpty(dependencies)) { + if (ServiceUtils.isEmpty(dependencies)) { return tokens; } for (String dependency : dependencies) { @@ -455,8 +455,8 @@ public class Component implements EventHandler<ComponentEvent> { if (instance.getContainerStatus() == null) { continue; } - if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) || - SliderUtils.isUnset(instance.getContainerStatus().getHost())) { + if (ServiceUtils.isEmpty(instance.getContainerStatus().getIPs()) || + ServiceUtils.isUnset(instance.getContainerStatus().getHost())) { continue; } String ip = instance.getContainerStatus().getIPs().get(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java index 2492454..9e5f98c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.BoundedAppender; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus; import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; @@ -503,7 +503,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>, + nodeId + ", will try again", e); return; } - if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils + if (ServiceUtils.isEmpty(status.getIPs()) || ServiceUtils .isUnset(status.getHost())) { return; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java index e4eae20..2d7c3bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.utils.CoreFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -132,7 +132,7 @@ public class AbstractLauncher { */ public ContainerLaunchContext completeContainerLaunch() throws IOException { - String cmdStr = SliderUtils.join(commands, " ", false); + String cmdStr = ServiceUtils.join(commands, " ", false); log.debug("Completed setting up container command {}", cmdStr); containerLaunchContext.setCommands(commands); @@ -205,7 +205,7 @@ public class AbstractLauncher { String key = entry.getKey(); LocalResource val = entry.getValue(); - log.debug(key + "=" + SliderUtils.stringify(val.getResource())); + log.debug(key + "=" + ServiceUtils.stringify(val.getResource())); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java index 22b3877..711abb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java @@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import java.util.ArrayList; import java.util.Arrays; @@ -63,7 +63,7 @@ public class ClasspathConstructor { } public String buildClasspath() { - return SliderUtils.join(pathElements, + return ServiceUtils.join(pathElements, CLASS_PATH_SEPARATOR, false); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java index 7baa284..5ed56e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java @@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.containerlaunch; import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import java.util.ArrayList; import java.util.List; @@ -81,6 +81,6 @@ public class CommandLineBuilder { * @return the command line */ public String build() { - return SliderUtils.join(argumentList, " "); + return ServiceUtils.join(argumentList, " "); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java index cbcb0d6..7f6cc04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java @@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.service.containerlaunch; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.exceptions.BadConfigException; import java.util.Map; @@ -53,7 +53,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder { * trimmed. */ public void setJVMHeap(String heap) { - if (SliderUtils.isSet(heap)) { + if (ServiceUtils.isSet(heap)) { add("-Xmx" + heap.trim()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java index 1923086..1ed13a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java @@ -21,7 +21,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +75,7 @@ public class HttpProbe extends Probe { public ProbeStatus ping(ComponentInstance instance) { ProbeStatus status = new ProbeStatus(); ContainerStatus containerStatus = instance.getContainerStatus(); - if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs()) + if (containerStatus == null || ServiceUtils.isEmpty(containerStatus.getIPs()) || StringUtils.isEmpty(containerStatus.getHost())) { status.fail(this, new IOException("IP is not available yet")); return status; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java index aba5859..85569f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.service.monitor.probe; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +67,7 @@ public class PortProbe extends Probe { public ProbeStatus ping(ComponentInstance instance) { ProbeStatus status = new ProbeStatus(); - if (instance.getContainerStatus() == null || SliderUtils + if (instance.getContainerStatus() == null || ServiceUtils .isEmpty(instance.getContainerStatus().getIPs())) { status.fail(this, new IOException( instance.getCompInstanceName() + ": IP is not available yet")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java index 75c647b..fc8953c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java @@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.ConfigFile; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import java.io.IOException; import java.nio.file.Paths; @@ -53,12 +53,12 @@ public abstract class AbstractClientProvider { public static final Set<String> createApplicationTags(String appName, String appVersion, String appDescription) { Set<String> tags = new HashSet<>(); - tags.add(SliderUtils.createNameTag(appName)); + tags.add(ServiceUtils.createNameTag(appName)); if (appVersion != null) { - tags.add(SliderUtils.createVersionTag(appVersion)); + tags.add(ServiceUtils.createVersionTag(appVersion)); } if (appDescription != null) { - tags.add(SliderUtils.createDescriptionTag(appDescription)); + tags.add(ServiceUtils.createDescriptionTag(appDescription)); } return tags; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java index 6ffb84d..6d74061 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder; @@ -70,7 +70,7 @@ public abstract class AbstractProviderService implements ProviderService, .initCompTokensForSubstitute(instance); tokensForSubstitution.putAll(globalTokens); // Set the environment variables in launcher - launcher.putEnv(SliderUtils + launcher.putEnv(ServiceUtils .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java index 582c5ac..e074dd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java @@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.apache.hadoop.yarn.service.utils.PublishedConfiguration; import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,7 +94,7 @@ public class ProviderUtils implements YarnServiceConstants { IOException, SliderException { try { - SliderUtils.putJar(providerResources, + ServiceUtils.putJar(providerResources, sliderFileSystem, providerClass, tempPath, @@ -127,14 +127,14 @@ public class ProviderUtils implements YarnServiceConstants { String libDir, String libLocalSrcDir) throws IOException, SliderException { - if (SliderUtils.isSet(libLocalSrcDir)) { + if (ServiceUtils.isSet(libLocalSrcDir)) { File file = new File(libLocalSrcDir); if (!file.exists() || !file.isDirectory()) { throw new BadCommandArgumentsException( "Supplied lib src dir %s is not valid", libLocalSrcDir); } } - SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, + ServiceUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libLocalSrcDir); } @@ -174,7 +174,7 @@ public class ProviderUtils implements YarnServiceConstants { Configuration conf = service.getConfiguration(); String keytabPathOnHost = conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { + if (ServiceUtils.isUnset(keytabPathOnHost)) { String amKeytabName = conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME); String keytabDir = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java index d418b59..cecca5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java @@ -29,7 +29,7 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,9 +70,9 @@ public class YarnRegistryViewForProviders { Preconditions.checkArgument(registryOperations != null, "null registry operations"); Preconditions.checkArgument(user != null, "null user"); - Preconditions.checkArgument(SliderUtils.isSet(serviceClass), + Preconditions.checkArgument(ServiceUtils.isSet(serviceClass), "unset service class"); - Preconditions.checkArgument(SliderUtils.isSet(instanceName), + Preconditions.checkArgument(ServiceUtils.isSet(instanceName), "instanceName"); Preconditions.checkArgument(applicationAttemptId != null, "null applicationAttemptId"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java index 281e1dfe..6b9d4d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java @@ -409,7 +409,7 @@ public class CoreFileSystem { */ public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName) throws IOException, SliderException { - File localFile = SliderUtils.findContainingJarOrFail(clazz); + File localFile = ServiceUtils.findContainingJarOrFail(clazz); return submitFile(localFile, tempPath, subdir, jarName); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java index 2dbf37f..1d64ed6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java @@ -85,7 +85,7 @@ public class PortScanner { if (remainingPortsToCheck != null) { return getAvailablePortViaPortArray(); } else { - return SliderUtils.getOpenPort(); + return ServiceUtils.getOpenPort(); } } @@ -95,7 +95,7 @@ public class PortScanner { Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator(); while (portsToCheck.hasNext() && !found) { int portToCheck = portsToCheck.next(); - found = SliderUtils.isPortAvailable(portToCheck); + found = ServiceUtils.isPortAvailable(portToCheck); if (found) { availablePort = portToCheck; portsToCheck.remove(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index 68db0bb..b58cea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -369,7 +369,7 @@ public class ServiceApiUtil { continue; } boolean dependenciesAlreadySorted = true; - if (!SliderUtils.isEmpty(component.getDependencies())) { + if (!ServiceUtils.isEmpty(component.getDependencies())) { for (String dependency : component.getDependencies()) { if (!sortedComponents.containsKey(dependency)) { dependenciesAlreadySorted = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java new file mode 100644 index 0000000..e18bcae --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URL; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; + +/** + * These are slider-specific Util methods + */ +public final class ServiceUtils { + + private static final Logger log = LoggerFactory.getLogger(ServiceUtils.class); + + private ServiceUtils() { + } + + /** + * Implementation of set-ness, groovy definition of true/false for a string + * @param s string + * @return true iff the string is neither null nor empty + */ + public static boolean isUnset(String s) { + return s == null || s.isEmpty(); + } + + public static boolean isSet(String s) { + return !isUnset(s); + } + + public static boolean isEmpty(Collection l) { + return l == null || l.isEmpty(); + } + + /** + * Find a containing JAR + * @param clazz class to find + * @return the file + * @throws IOException any IO problem, including the class not having a + * classloader + * @throws FileNotFoundException if the class did not resolve to a file + */ + public static File findContainingJarOrFail(Class clazz) throws IOException { + File localFile = ServiceUtils.findContainingJar(clazz); + if (null == localFile) { + throw new FileNotFoundException("Could not find JAR containing " + clazz); + } + return localFile; + } + + + /** + * Find a containing JAR + * @param my_class class to find + * @return the file or null if it is not found + * @throws IOException any IO problem, including the class not having a + * classloader + */ + public static File findContainingJar(Class my_class) throws IOException { + ClassLoader loader = my_class.getClassLoader(); + if (loader == null) { + throw new IOException( + "Class " + my_class + " does not have a classloader!"); + } + String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; + Enumeration<URL> urlEnumeration = loader.getResources(class_file); + for (; urlEnumeration.hasMoreElements(); ) { + URL url = urlEnumeration.nextElement(); + if ("jar".equals(url.getProtocol())) { + String toReturn = url.getPath(); + if (toReturn.startsWith("file:")) { + toReturn = toReturn.substring("file:".length()); + } + // URLDecoder is a misnamed class, since it actually decodes + // x-www-form-urlencoded MIME type rather than actual + // URL encoding (which the file path has). Therefore it would + // decode +s to ' 's which is incorrect (spaces are actually + // either unencoded or encoded as "%20"). Replace +s first, so + // that they are kept sacred during the decoding process. + toReturn = toReturn.replaceAll("\\+", "%2B"); + toReturn = URLDecoder.decode(toReturn, "UTF-8"); + String jarFilePath = toReturn.replaceAll("!.*$", ""); + return new File(jarFilePath); + } else { + log.info("could not locate JAR containing {} URL={}", my_class, url); + } + } + return null; + } + + /** + * Copy a file to a new FS -both paths must be qualified. + * @param conf conf file + * @param srcFile src file + * @param destFile dest file + */ + @SuppressWarnings("deprecation") + public static void copy(Configuration conf, + Path srcFile, + Path destFile) throws + IOException, + BadClusterStateException { + FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); + //list all paths in the src. + if (!srcFS.exists(srcFile)) { + throw new FileNotFoundException("Source file not found " + srcFile); + } + if (!srcFS.isFile(srcFile)) { + throw new FileNotFoundException( + "Source file not a file " + srcFile); + } + FileSystem destFS = FileSystem.get(destFile.toUri(), conf); + FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); + } + + /** + * Take a collection, return a list containing the string value of every + * element in the collection. + * @param c collection + * @return a stringified list + */ + public static List<String> collectionToStringList(Collection c) { + List<String> l = new ArrayList<>(c.size()); + for (Object o : c) { + l.add(o.toString()); + } + return l; + } + + /** + * Join an collection of objects with a separator that appears after every + * instance in the list -including at the end + * @param collection collection to call toString() on each element + * @param separator separator string + * @return the joined entries + */ + public static String join(Collection collection, String separator) { + return join(collection, separator, true); + } + + /** + * Join an collection of objects with a separator that appears after every + * instance in the list -optionally at the end + * @param collection collection to call toString() on each element + * @param separator separator string + * @param trailing add a trailing entry or not + * @return the joined entries + */ + public static String join(Collection collection, + String separator, + boolean trailing) { + StringBuilder b = new StringBuilder(); + // fast return on empty collection + if (collection.isEmpty()) { + return trailing ? separator : ""; + } + for (Object o : collection) { + b.append(o); + b.append(separator); + } + int length = separator.length(); + String s = b.toString(); + return (trailing || s.isEmpty()) ? + s : (b.substring(0, b.length() - length)); + } + + /** + * Join an array of strings with a separator that appears after every + * instance in the list -optionally at the end + * @param collection strings + * @param separator separator string + * @param trailing add a trailing entry or not + * @return the joined entries + */ + public static String join(String[] collection, String separator, + boolean trailing) { + return join(Arrays.asList(collection), separator, trailing); + } + + /** + * Resolve a mandatory environment variable + * @param key env var + * @return the resolved value + * @throws BadClusterStateException + */ + public static String mandatoryEnvVariable(String key) throws + BadClusterStateException { + String v = System.getenv(key); + if (v == null) { + throw new BadClusterStateException("Missing Environment variable " + key); + } + return v; + } + + /** + * Generic map merge logic + * @param first first map + * @param second second map + * @param <T1> key type + * @param <T2> value type + * @return 'first' merged with the second + */ + public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first, + Map<T1, T2> second) { + Preconditions.checkArgument(first != null, "Null 'first' value"); + Preconditions.checkArgument(second != null, "Null 'second' value"); + for (Map.Entry<T1, T2> entry : second.entrySet()) { + T1 key = entry.getKey(); + if (!first.containsKey(key)) { + first.put(key, entry.getValue()); + } + } + return first; + } + + /** + * Convert a map to a multi-line string for printing + * @param map map to stringify + * @return a string representation of the map + */ + public static String stringifyMap(Map<String, String> map) { + StringBuilder builder = new StringBuilder(); + for (Map.Entry<String, String> entry : map.entrySet()) { + builder.append(entry.getKey()) + .append("=\"") + .append(entry.getValue()) + .append("\"\n"); + + } + return builder.toString(); + } + + /** + * Convert a YARN URL into a string value of a normal URL + * @param url URL + * @return string representatin + */ + public static String stringify(org.apache.hadoop.yarn.api.records.URL url) { + StringBuilder builder = new StringBuilder(); + builder.append(url.getScheme()).append("://"); + if (url.getHost() != null) { + builder.append(url.getHost()).append(":").append(url.getPort()); + } + builder.append(url.getFile()); + return builder.toString(); + } + + /** + * Get a random open port + * @return true if the port was available for listening on + */ + public static int getOpenPort() throws IOException { + ServerSocket socket = null; + try { + socket = new ServerSocket(0); + return socket.getLocalPort(); + } finally { + if (socket != null) { + socket.close(); + } + } + } + + /** + * See if a port is available for listening on by trying to listen + * on it and seeing if that works or fails. + * @param port port to listen to + * @return true if the port was available for listening on + */ + public static boolean isPortAvailable(int port) { + try { + ServerSocket socket = new ServerSocket(port); + socket.close(); + return true; + } catch (IOException e) { + return false; + } + } + + // Build env map: key -> value; + // value will be replaced by the corresponding value in tokenMap, if any. + public static Map<String, String> buildEnvMap( + org.apache.hadoop.yarn.service.api.records.Configuration conf, + Map<String,String> tokenMap) { + if (tokenMap == null) { + return conf.getEnv(); + } + Map<String, String> env = new HashMap<>(); + for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + for (Map.Entry<String,String> token : tokenMap.entrySet()) { + val = val.replaceAll(Pattern.quote(token.getKey()), + token.getValue()); + } + env.put(key,val); + } + return env; + } + + public static String[] getLibDirs() { + String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR); + if (isUnset(libDirStr)) { + return ArrayUtils.EMPTY_STRING_ARRAY; + } + return StringUtils.split(libDirStr, ','); + } + + /** + * Submit a JAR containing a specific class and map it + * @param providerResources provider map to build up + * @param sliderFileSystem remote fs + * @param clazz class to look for + * @param libdir lib directory + * @param jarName <i>At the destination</i> + * @return the local resource ref + * @throws IOException trouble copying to HDFS + */ + public static LocalResource putJar(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Class clazz, + Path tempPath, + String libdir, + String jarName + ) + throws IOException, SliderException { + LocalResource res = sliderFileSystem.submitJarWithClass( + clazz, + tempPath, + libdir, + jarName); + providerResources.put(libdir + "/" + jarName, res); + return res; + } + + /** + * Submit a JAR containing and map it + * @param providerResources provider map to build up + * @param sliderFileSystem remote fs + * @param libDir lib directory + * @param srcPath copy jars from + */ + public static void putAllJars(Map<String, LocalResource> providerResources, + SliderFileSystem sliderFileSystem, + Path tempPath, + String libDir, + String srcPath) throws IOException, SliderException { + log.debug("Loading all dependencies from {}", srcPath); + if (ServiceUtils.isSet(srcPath)) { + File srcFolder = new File(srcPath); + FilenameFilter jarFilter = createJarFilter(); + File[] listOfJars = srcFolder.listFiles(jarFilter); + if (listOfJars == null || listOfJars.length == 0) { + return; + } + for (File jarFile : listOfJars) { + LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); + providerResources.put(libDir + "/" + jarFile.getName(), res); + } + } + } + + /** + * Accept all filenames ending with {@code .jar} + * @return a filename filter + */ + public static FilenameFilter createJarFilter() { + return new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.toLowerCase(Locale.ENGLISH).endsWith(".jar"); + } + }; + } + + /** + * Create a file:// path from a local file + * @param file file to point the path + * @return a new Path + */ + public static Path createLocalPath(File file) { + return new Path(file.toURI()); + } + + /** + * Build up the classpath for execution + * -behaves very differently on a mini test cluster vs a production + * production one. + * + * @param sliderConfDir relative path to the dir containing slider config + * options to put on the classpath -or null + * @param libdir directory containing the JAR files + * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use + * (and hence the current classpath should be used, not anything built up) + * @return a classpath + */ + public static ClasspathConstructor buildClasspath(String sliderConfDir, + String libdir, + SliderFileSystem sliderFileSystem, + boolean usingMiniMRCluster) { + + ClasspathConstructor classpath = new ClasspathConstructor(); + classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME); + + // add the runtime classpath needed for tests to work + if (usingMiniMRCluster) { + // for mini cluster we pass down the java CP properties + // and nothing else + classpath.appendAll(classpath.localJVMClasspath()); + } else { + if (sliderConfDir != null) { + classpath.addClassDirectory(sliderConfDir); + } + classpath.addLibDir(libdir); + if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) { + classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK); + } + classpath.addRemoteClasspathEnvVar(); + classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$()); + } + return classpath; + } + + /** + * Given a source folder create a tar.gz file + * + * @param libDirs + * @param tarGzipFile + * + * @throws IOException + */ + public static void tarGzipFolder(String[] libDirs, File tarGzipFile, + FilenameFilter filter) throws IOException { + log.info("Tar-gzipping folders {} to {}", libDirs, + tarGzipFile.getAbsolutePath()); + + try(TarArchiveOutputStream taos = + new TarArchiveOutputStream(new GZIPOutputStream( + new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) { + for (String libDir : libDirs) { + File srcFolder = new File(libDir); + List<String> files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true, filter); + for (String file : files) { + File srcFile = new File(srcFolder, file); + TarArchiveEntry tarEntry = new TarArchiveEntry( + srcFile, file); + taos.putArchiveEntry(tarEntry); + try(FileInputStream in = new FileInputStream(srcFile)) { + org.apache.commons.io.IOUtils.copy(in, taos); + } + taos.flush(); + taos.closeArchiveEntry(); + } + } + } + } + + private static void generateFileList(List<String> fileList, File node, + File rootFolder, Boolean relative, FilenameFilter filter) { + if (node.isFile()) { + String fileFullPath = node.toString(); + if (relative) { + fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, + fileFullPath.length())); + } else { + fileList.add(fileFullPath); + } + } + + if (node.isDirectory()) { + String[] subNode = node.list(filter); + if (subNode == null || subNode.length == 0) { + return; + } + for (String filename : subNode) { + generateFileList(fileList, new File(node, filename), rootFolder, + relative, filter); + } + } + } + + public static String createNameTag(String name) { + return "Name: " + name; + } + + public static String createVersionTag(String version) { + return "Version: " + version; + } + + public static String createDescriptionTag(String description) { + return "Description: " + description; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java deleted file mode 100644 index 2809dfb..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java +++ /dev/null @@ -1,564 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.base.Preconditions; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; -import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; -import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.net.ServerSocket; -import java.net.URL; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.zip.GZIPOutputStream; - -/** - * These are slider-specific Util methods - */ -public final class SliderUtils { - - private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); - - private SliderUtils() { - } - - /** - * Implementation of set-ness, groovy definition of true/false for a string - * @param s string - * @return true iff the string is neither null nor empty - */ - public static boolean isUnset(String s) { - return s == null || s.isEmpty(); - } - - public static boolean isSet(String s) { - return !isUnset(s); - } - - public static boolean isEmpty(Collection l) { - return l == null || l.isEmpty(); - } - - /** - * Find a containing JAR - * @param clazz class to find - * @return the file - * @throws IOException any IO problem, including the class not having a - * classloader - * @throws FileNotFoundException if the class did not resolve to a file - */ - public static File findContainingJarOrFail(Class clazz) throws IOException { - File localFile = SliderUtils.findContainingJar(clazz); - if (null == localFile) { - throw new FileNotFoundException("Could not find JAR containing " + clazz); - } - return localFile; - } - - - /** - * Find a containing JAR - * @param my_class class to find - * @return the file or null if it is not found - * @throws IOException any IO problem, including the class not having a - * classloader - */ - public static File findContainingJar(Class my_class) throws IOException { - ClassLoader loader = my_class.getClassLoader(); - if (loader == null) { - throw new IOException( - "Class " + my_class + " does not have a classloader!"); - } - String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; - Enumeration<URL> urlEnumeration = loader.getResources(class_file); - for (; urlEnumeration.hasMoreElements(); ) { - URL url = urlEnumeration.nextElement(); - if ("jar".equals(url.getProtocol())) { - String toReturn = url.getPath(); - if (toReturn.startsWith("file:")) { - toReturn = toReturn.substring("file:".length()); - } - // URLDecoder is a misnamed class, since it actually decodes - // x-www-form-urlencoded MIME type rather than actual - // URL encoding (which the file path has). Therefore it would - // decode +s to ' 's which is incorrect (spaces are actually - // either unencoded or encoded as "%20"). Replace +s first, so - // that they are kept sacred during the decoding process. - toReturn = toReturn.replaceAll("\\+", "%2B"); - toReturn = URLDecoder.decode(toReturn, "UTF-8"); - String jarFilePath = toReturn.replaceAll("!.*$", ""); - return new File(jarFilePath); - } else { - log.info("could not locate JAR containing {} URL={}", my_class, url); - } - } - return null; - } - - /** - * Copy a file to a new FS -both paths must be qualified. - * @param conf conf file - * @param srcFile src file - * @param destFile dest file - */ - @SuppressWarnings("deprecation") - public static void copy(Configuration conf, - Path srcFile, - Path destFile) throws - IOException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); - //list all paths in the src. - if (!srcFS.exists(srcFile)) { - throw new FileNotFoundException("Source file not found " + srcFile); - } - if (!srcFS.isFile(srcFile)) { - throw new FileNotFoundException( - "Source file not a file " + srcFile); - } - FileSystem destFS = FileSystem.get(destFile.toUri(), conf); - FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); - } - - /** - * Take a collection, return a list containing the string value of every - * element in the collection. - * @param c collection - * @return a stringified list - */ - public static List<String> collectionToStringList(Collection c) { - List<String> l = new ArrayList<>(c.size()); - for (Object o : c) { - l.add(o.toString()); - } - return l; - } - - /** - * Join an collection of objects with a separator that appears after every - * instance in the list -including at the end - * @param collection collection to call toString() on each element - * @param separator separator string - * @return the joined entries - */ - public static String join(Collection collection, String separator) { - return join(collection, separator, true); - } - - /** - * Join an collection of objects with a separator that appears after every - * instance in the list -optionally at the end - * @param collection collection to call toString() on each element - * @param separator separator string - * @param trailing add a trailing entry or not - * @return the joined entries - */ - public static String join(Collection collection, - String separator, - boolean trailing) { - StringBuilder b = new StringBuilder(); - // fast return on empty collection - if (collection.isEmpty()) { - return trailing ? separator : ""; - } - for (Object o : collection) { - b.append(o); - b.append(separator); - } - int length = separator.length(); - String s = b.toString(); - return (trailing || s.isEmpty()) ? - s : (b.substring(0, b.length() - length)); - } - - /** - * Join an array of strings with a separator that appears after every - * instance in the list -optionally at the end - * @param collection strings - * @param separator separator string - * @param trailing add a trailing entry or not - * @return the joined entries - */ - public static String join(String[] collection, String separator, - boolean trailing) { - return join(Arrays.asList(collection), separator, trailing); - } - - /** - * Resolve a mandatory environment variable - * @param key env var - * @return the resolved value - * @throws BadClusterStateException - */ - public static String mandatoryEnvVariable(String key) throws - BadClusterStateException { - String v = System.getenv(key); - if (v == null) { - throw new BadClusterStateException("Missing Environment variable " + key); - } - return v; - } - - /** - * Generic map merge logic - * @param first first map - * @param second second map - * @param <T1> key type - * @param <T2> value type - * @return 'first' merged with the second - */ - public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first, - Map<T1, T2> second) { - Preconditions.checkArgument(first != null, "Null 'first' value"); - Preconditions.checkArgument(second != null, "Null 'second' value"); - for (Map.Entry<T1, T2> entry : second.entrySet()) { - T1 key = entry.getKey(); - if (!first.containsKey(key)) { - first.put(key, entry.getValue()); - } - } - return first; - } - - /** - * Convert a map to a multi-line string for printing - * @param map map to stringify - * @return a string representation of the map - */ - public static String stringifyMap(Map<String, String> map) { - StringBuilder builder = new StringBuilder(); - for (Map.Entry<String, String> entry : map.entrySet()) { - builder.append(entry.getKey()) - .append("=\"") - .append(entry.getValue()) - .append("\"\n"); - - } - return builder.toString(); - } - - /** - * Convert a YARN URL into a string value of a normal URL - * @param url URL - * @return string representatin - */ - public static String stringify(org.apache.hadoop.yarn.api.records.URL url) { - StringBuilder builder = new StringBuilder(); - builder.append(url.getScheme()).append("://"); - if (url.getHost() != null) { - builder.append(url.getHost()).append(":").append(url.getPort()); - } - builder.append(url.getFile()); - return builder.toString(); - } - - /** - * Get a random open port - * @return true if the port was available for listening on - */ - public static int getOpenPort() throws IOException { - ServerSocket socket = null; - try { - socket = new ServerSocket(0); - return socket.getLocalPort(); - } finally { - if (socket != null) { - socket.close(); - } - } - } - - /** - * See if a port is available for listening on by trying to listen - * on it and seeing if that works or fails. - * @param port port to listen to - * @return true if the port was available for listening on - */ - public static boolean isPortAvailable(int port) { - try { - ServerSocket socket = new ServerSocket(port); - socket.close(); - return true; - } catch (IOException e) { - return false; - } - } - - // Build env map: key -> value; - // value will be replaced by the corresponding value in tokenMap, if any. - public static Map<String, String> buildEnvMap( - org.apache.hadoop.yarn.service.api.records.Configuration conf, - Map<String,String> tokenMap) { - if (tokenMap == null) { - return conf.getEnv(); - } - Map<String, String> env = new HashMap<>(); - for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) { - String key = entry.getKey(); - String val = entry.getValue(); - for (Map.Entry<String,String> token : tokenMap.entrySet()) { - val = val.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } - env.put(key,val); - } - return env; - } - - public static String[] getLibDirs() { - String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR); - if (isUnset(libDirStr)) { - return ArrayUtils.EMPTY_STRING_ARRAY; - } - return StringUtils.split(libDirStr, ','); - } - - /** - * Submit a JAR containing a specific class and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - * @param clazz class to look for - * @param libdir lib directory - * @param jarName <i>At the destination</i> - * @return the local resource ref - * @throws IOException trouble copying to HDFS - */ - public static LocalResource putJar(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Class clazz, - Path tempPath, - String libdir, - String jarName - ) - throws IOException, SliderException { - LocalResource res = sliderFileSystem.submitJarWithClass( - clazz, - tempPath, - libdir, - jarName); - providerResources.put(libdir + "/" + jarName, res); - return res; - } - - /** - * Submit a JAR containing and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - * @param libDir lib directory - * @param srcPath copy jars from - */ - public static void putAllJars(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libDir, - String srcPath) throws IOException, SliderException { - log.info("Loading all dependencies from {}", srcPath); - if (SliderUtils.isSet(srcPath)) { - File srcFolder = new File(srcPath); - FilenameFilter jarFilter = createJarFilter(); - File[] listOfJars = srcFolder.listFiles(jarFilter); - if (listOfJars == null || listOfJars.length == 0) { - return; - } - for (File jarFile : listOfJars) { - LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); - providerResources.put(libDir + "/" + jarFile.getName(), res); - } - } - } - - /** - * Accept all filenames ending with {@code .jar} - * @return a filename filter - */ - public static FilenameFilter createJarFilter() { - return new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.toLowerCase(Locale.ENGLISH).endsWith(".jar"); - } - }; - } - - /** - * Submit the AM tar.gz containing all dependencies and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - */ - public static void putAmTarGzipAndUpdate( - Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem - ) throws IOException, SliderException { - log.info("Loading all dependencies from {}{}", - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME, - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); - sliderFileSystem.submitTarGzipAndUpdate(providerResources); - } - - /** - * Create a file:// path from a local file - * @param file file to point the path - * @return a new Path - */ - public static Path createLocalPath(File file) { - return new Path(file.toURI()); - } - - /** - * Build up the classpath for execution - * -behaves very differently on a mini test cluster vs a production - * production one. - * - * @param sliderConfDir relative path to the dir containing slider config - * options to put on the classpath -or null - * @param libdir directory containing the JAR files - * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use - * (and hence the current classpath should be used, not anything built up) - * @return a classpath - */ - public static ClasspathConstructor buildClasspath(String sliderConfDir, - String libdir, - SliderFileSystem sliderFileSystem, - boolean usingMiniMRCluster) { - - ClasspathConstructor classpath = new ClasspathConstructor(); - classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME); - - // add the runtime classpath needed for tests to work - if (usingMiniMRCluster) { - // for mini cluster we pass down the java CP properties - // and nothing else - classpath.appendAll(classpath.localJVMClasspath()); - } else { - if (sliderConfDir != null) { - classpath.addClassDirectory(sliderConfDir); - } - classpath.addLibDir(libdir); - if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) { - classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK); - } else { - log.info( - "For faster submission of apps, upload dependencies using cmd " + - "enableFastLaunch"); - } - classpath.addRemoteClasspathEnvVar(); - classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$()); - } - return classpath; - } - - /** - * Given a source folder create a tar.gz file - * - * @param libDirs - * @param tarGzipFile - * - * @throws IOException - */ - public static void tarGzipFolder(String[] libDirs, File tarGzipFile, - FilenameFilter filter) throws IOException { - log.info("Tar-gzipping folders {} to {}", libDirs, - tarGzipFile.getAbsolutePath()); - - try(TarArchiveOutputStream taos = - new TarArchiveOutputStream(new GZIPOutputStream( - new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) { - for (String libDir : libDirs) { - File srcFolder = new File(libDir); - List<String> files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true, filter); - for (String file : files) { - File srcFile = new File(srcFolder, file); - TarArchiveEntry tarEntry = new TarArchiveEntry( - srcFile, file); - taos.putArchiveEntry(tarEntry); - try(FileInputStream in = new FileInputStream(srcFile)) { - org.apache.commons.io.IOUtils.copy(in, taos); - } - taos.flush(); - taos.closeArchiveEntry(); - } - } - } - } - - private static void generateFileList(List<String> fileList, File node, - File rootFolder, Boolean relative, FilenameFilter filter) { - if (node.isFile()) { - String fileFullPath = node.toString(); - if (relative) { - fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, - fileFullPath.length())); - } else { - fileList.add(fileFullPath); - } - } - - if (node.isDirectory()) { - String[] subNode = node.list(filter); - if (subNode == null || subNode.length == 0) { - return; - } - for (String filename : subNode) { - generateFileList(fileList, new File(node, filename), rootFolder, - relative, filter); - } - } - } - - public static String createNameTag(String name) { - return "Name: " + name; - } - - public static String createVersionTag(String version) { - return "Version: " + version; - } - - public static String createDescriptionTag(String description) { - return "Description: " + description; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java index 1fa07ce..c6e8525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java @@ -32,7 +32,7 @@ public class ZookeeperUtils { String zkPort = Integer.toString(port); //parse the hosts String[] hostlist = zkHosts.split(",", 0); - String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false); + String quorum = ServiceUtils.join(hostlist, ":" + zkPort + ",", false); return quorum; } @@ -117,7 +117,7 @@ public class ZookeeperUtils { for (HostAndPort hostAndPort : hostAndPorts) { entries.add(buildQuorumEntry(hostAndPort, defaultPort)); } - return SliderUtils.join(entries, ",", false); + return ServiceUtils.join(entries, ",", false); } public static String convertToHostsOnlyList(String quorum) throws http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java index 6159215..d7fa9a04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.junit.Assert; import org.junit.Test; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
