http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java deleted file mode 100644 index 204ad9a..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -/** - * Actions. - * Only some of these are supported by specific Slider Services; they - * are listed here to ensure the names are consistent - */ -public interface SliderActions { - String ACTION_AM_SUICIDE = "am-suicide"; - String ACTION_BUILD = "build"; - String ACTION_CLIENT = "client"; - String ACTION_CREATE = "create"; - String ACTION_DIAGNOSTICS = "diagnostics"; - String ACTION_DEPENDENCY = "dependency"; - String ACTION_UPDATE = "update"; - String ACTION_UPGRADE = "upgrade"; - String ACTION_DESTROY = "destroy"; - String ACTION_ECHO = "echo"; - String ACTION_EXISTS = "exists"; - String ACTION_FLEX = "flex"; - String ACTION_FREEZE = "stop"; - String ACTION_HELP = "help"; - String ACTION_INSTALL_KEYTAB = "install-keytab"; - String ACTION_INSTALL_PACKAGE = "install-package"; - String ACTION_KDIAG = "kdiag"; - String ACTION_KEYTAB = "keytab"; - String ACTION_KILL_CONTAINER = "kill-container"; - String ACTION_LIST = "list"; - String ACTION_LOOKUP = "lookup"; - String ACTION_NODES = "nodes"; - String ACTION_PACKAGE = "package"; - String ACTION_PREFLIGHT = "preflight"; - String ACTION_RECONFIGURE = "reconfigure"; - String ACTION_REGISTRY = "registry"; - String ACTION_RESOLVE = "resolve"; - String ACTION_RESOURCE = "resource"; - String ACTION_STATUS = "status"; - String ACTION_THAW = "start"; - String ACTION_TOKENS = "tokens"; - - String ACTION_VERSION = "version"; - String DESCRIBE_ACTION_AM_SUICIDE = - "Tell the Slider Application Master to simulate a process failure by terminating itself"; - String DESCRIBE_ACTION_BUILD = - "Build a Slider cluster specification, but do not start it"; - String DESCRIBE_ACTION_CREATE = - "Create a live Slider application"; - String DESCRIBE_ACTION_DEPENDENCY = - "Slider AM and agent dependency (libraries) management"; - String DESCRIBE_ACTION_UPDATE = - "Update template for a Slider application"; - String DESCRIBE_ACTION_UPGRADE = - "Rolling upgrade/downgrade the application to a newer/previous version"; - String DESCRIBE_ACTION_DESTROY = - "Destroy a stopped Slider application"; - String DESCRIBE_ACTION_EXISTS = - "Probe for an application running"; - String DESCRIBE_ACTION_FLEX = "Flex a Slider application"; - String DESCRIBE_ACTION_FREEZE = - "Stop a running application"; - String DESCRIBE_ACTION_GETCONF = - "Get the configuration of an application"; - String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems"; - String DESCRIBE_ACTION_KILL_CONTAINER = - "Kill a container in the application"; - String DESCRIBE_ACTION_HELP = "Print help information"; - String DESCRIBE_ACTION_LIST = - "List running Slider applications"; - String DESCRIBE_ACTION_LOOKUP = - "look up a YARN application"; - String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application"; - String DESCRIBE_ACTION_MONITOR = - "Monitor a running application"; - String DESCRIBE_ACTION_REGISTRY = - "Query the registry of a YARN application"; - String DESCRIBE_ACTION_RESOLVE = - "Resolve or list records in the YARN registry"; - String DESCRIBE_ACTION_STATUS = - "Get the status of an application"; - String DESCRIBE_ACTION_THAW = - "Start a stopped application"; - String DESCRIBE_ACTION_VERSION = - "Print the Slider version information"; - String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." + - " Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."; - String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages"; - String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore"; - String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." + - " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; - String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; - String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; - String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory"; - -} -
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java deleted file mode 100644 index b0d1ebf..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.converters.BaseConverter; - -import java.net.URI; -import java.net.URISyntaxException; - -public class URIArgumentConverter extends BaseConverter<URI> { - - public URIArgumentConverter(String optionName) { - super(optionName); - } - - @Override - public URI convert(String value) { - try { - return new URI(value); - } catch (URISyntaxException e) { - throw new RuntimeException("Cannot make a URI from " + value); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java deleted file mode 100644 index 8894309..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.slider.common.params; - -import com.beust.jcommander.converters.BaseConverter; - -import java.net.MalformedURLException; -import java.net.URL; - -public class URLArgumentConverter extends BaseConverter<URL> { - public URLArgumentConverter(String optionName) { - super(optionName); - } - - @Override - public URL convert(String value) { - try { - return new URL(value); - } catch (MalformedURLException e) { - throw new RuntimeException("Cannot make a URL from " + value); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java deleted file mode 100644 index 1c27c01..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; - -public class WaitArgsDelegate extends AbstractArgsDelegate implements - WaitTimeAccessor { - - - //--wait [timeout] - @Parameter(names = {ARG_WAIT}, - description = "time to wait for an action to complete") - public int waittime = 0; - - - @Override - public int getWaittime() { - return waittime; - } - - @Override - public void setWaittime(int waittime) { - this.waittime = waittime; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java deleted file mode 100644 index 13d4d5a..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -public interface WaitTimeAccessor { - int getWaittime(); - void setWaittime(int waittime); -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java deleted file mode 100644 index 6380d0c..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import java.io.Serializable; -import java.util.Comparator; - -/** - * Some general comparators - */ -public class Comparators { - - public static class LongComparator implements Comparator<Long>, Serializable { - @Override - public int compare(Long o1, Long o2) { - long result = o1 - o2; - // need to comparisons with a diff greater than integer size - if (result < 0 ) { - return -1; - } else if (result > 0) { - return 1; - } - return 0; - } - } -public static class InvertedLongComparator implements Comparator<Long>, Serializable { - private static final LongComparator inner = new LongComparator(); - @Override - public int compare(Long o1, Long o2) { - return -inner.compare(o1, o2); - } - } - - - /** - * Little template class to reverse any comparitor - * @param <CompareType> the type that is being compared - */ - public static class ComparatorReverser<CompareType> implements Comparator<CompareType>, - Serializable { - - final Comparator<CompareType> instance; - - public ComparatorReverser(Comparator<CompareType> instance) { - this.instance = instance; - } - - @Override - public int compare(CompareType first, CompareType second) { - return instance.compare(second, first); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java deleted file mode 100644 index 9db241d..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ /dev/null @@ -1,658 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.core.exceptions.BadConfigException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Methods to aid in config, both in the Configuration class and - * with other parts of setting up Slider-initated processes. - * - * Some of the methods take an argument of a map iterable for their sources; this allows - * the same method - */ -public class ConfigHelper { - private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); - - private static AtomicBoolean sliderResourceInjected = - new AtomicBoolean(false); - private static AtomicBoolean sliderResourceInjectionAttempted = - new AtomicBoolean(false); - - /** - * Dump the (sorted) configuration - * @param conf config - * @return the sorted keyset - */ - public static Set<String> dumpConf(Configuration conf) { - Set<String> keys = sortedConfigKeys(conf); - for (String key : keys) { - log.info("{}={}", key, conf.get(key)); - } - return keys; - } - - /** - * Take a configuration and return a sorted set - * @param conf config - * @return the sorted keyset - - */ - public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { - TreeSet<String> sorted = new TreeSet<String>(); - for (Map.Entry<String, String> entry : conf) { - sorted.add(entry.getKey()); - } - return sorted; - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Map<String, String> map, - String origin) throws BadConfigException { - addConfigMap(config, map.entrySet(), origin); - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Iterable<Map.Entry<String, String>> map, - String origin) throws BadConfigException { - for (Map.Entry<String, String> mapEntry : map) { - String key = mapEntry.getKey(); - String value = mapEntry.getValue(); - if (value == null) { - throw new BadConfigException("Null value for property " + key); - } - config.set(key, value, origin); - } - } - - - /** - * Save a config file in a destination directory on a given filesystem - * @param systemConf system conf used for creating filesystems - * @param confToSave config to save - * @param confdir the directory path where the file is to go - * @param filename the filename - * @return the destination path where the file was saved - * @throws IOException IO problems - */ - public static Path saveConfig(Configuration systemConf, - Configuration confToSave, - Path confdir, - String filename) throws IOException { - FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); - Path destPath = new Path(confdir, filename); - saveConfig(fs, destPath, confToSave); - return destPath; - } - - /** - * Save a config - * @param fs filesystem - * @param destPath dest to save - * @param confToSave config to save - * @throws IOException IO problems - */ - public static void saveConfig(FileSystem fs, - Path destPath, - Configuration confToSave) throws - IOException { - FSDataOutputStream fos = fs.create(destPath); - try { - confToSave.writeXml(fos); - } finally { - IOUtils.closeStream(fos); - } - } - - /** - * Convert to an XML string - * @param conf configuration - * @return conf - * @throws IOException - */ - public static String toXml(Configuration conf) throws IOException { - StringWriter writer = new StringWriter(); - conf.writeXml(writer); - return writer.toString(); - } - - /** - * This will load and parse a configuration to an XML document - * @param fs filesystem - * @param path path - * @return an XML document - * @throws IOException IO failure - */ - public Document parseConfiguration(FileSystem fs, - Path path) throws - IOException { - - - byte[] data = loadBytes(fs, path); - //this is here to track down a parse issue - //related to configurations - String s = new String(data, 0, data.length); - log.debug("XML resource {} is \"{}\"", path, s); -/* JDK7 - try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { - Document document = parseConfigXML(in); - return document; - } catch (ParserConfigurationException | SAXException e) { - throw new IOException(e); - } -*/ - ByteArrayInputStream in= null; - try { - in = new ByteArrayInputStream(data); - Document document = parseConfigXML(in); - return document; - } catch (ParserConfigurationException e) { - throw new IOException(e); - } catch (SAXException e) { - throw new IOException(e); - } finally { - IOUtils.closeStream(in); - } - } - - public static byte[] loadBytes(FileSystem fs, Path path) throws IOException { - int len = (int) fs.getLength(path); - byte[] data = new byte[len]; - /* JDK7 - try(FSDataInputStream in = fs.open(path)) { - in.readFully(0, data); - } -*/ - FSDataInputStream in = null; - in = fs.open(path); - try { - in.readFully(0, data); - } finally { - IOUtils.closeStream(in); - } - return data; - } - - /** - * Load a configuration from ANY FS path. The normal Configuration - * loader only works with file:// URIs - * @param fs filesystem - * @param path path - * @return a loaded resource - * @throws IOException - */ - public static Configuration loadConfiguration(FileSystem fs, - Path path) throws IOException { - byte[] data = loadBytes(fs, path); - - ByteArrayInputStream in2; - - in2 = new ByteArrayInputStream(data); - Configuration conf1 = new Configuration(false); - conf1.addResource(in2); - //now clone it while dropping all its sources - Configuration conf2 = new Configuration(false); - String src = path.toString(); - for (Map.Entry<String, String> entry : conf1) { - String key = entry.getKey(); - String value = entry.getValue(); - conf2.set(key, value, src); - } - return conf2; - } - - - /** - * Generate a config file in a destination directory on the local filesystem - * @param confdir the directory path where the file is to go - * @param filename the filename - * @return the destination path - */ - public static File saveConfig(Configuration generatingConf, - File confdir, - String filename) throws IOException { - - - File destPath = new File(confdir, filename); - OutputStream fos = new FileOutputStream(destPath); - try { - generatingConf.writeXml(fos); - } finally { - IOUtils.closeStream(fos); - } - return destPath; - } - - /** - * Parse an XML Hadoop configuration into an XML document. x-include - * is supported, but as the location isn't passed in, relative - * URIs are out. - * @param in instream - * @return a document - * @throws ParserConfigurationException parser feature problems - * @throws IOException IO problems - * @throws SAXException XML is invalid - */ - public static Document parseConfigXML(InputStream in) throws - ParserConfigurationException, - IOException, - SAXException { - DocumentBuilderFactory docBuilderFactory - = DocumentBuilderFactory.newInstance(); - //ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - - //allow includes in the xml file - docBuilderFactory.setNamespaceAware(true); - docBuilderFactory.setXIncludeAware(true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - return builder.parse(in); - } - - /** - * Load a Hadoop configuration from a local file. - * @param file file to load - * @return a configuration which hasn't actually had the load triggered - * yet. - * @throws FileNotFoundException file is not there - * @throws IOException any other IO problem - */ - public static Configuration loadConfFromFile(File file) throws - IOException { - return loadConfFromFile(file, false); - } - - /** - * - * Load a Hadoop configuration from a local file. - * @param file file to load - * @param loadDefaults flag to indicate if the defaults should be loaded yet - * @return a configuration which hasn't actually had the load triggered - * yet. - * @throws FileNotFoundException file is not there - * @throws IOException any other IO problem - */ - public static Configuration loadConfFromFile(File file, - boolean loadDefaults) throws IOException { - if (!file.exists()) { - throw new FileNotFoundException("File not found :" - + file.getAbsoluteFile()); - } - Configuration conf = new Configuration(loadDefaults); - try { - conf.addResource(file.toURI().toURL()); - } catch (MalformedURLException e) { - // should never happen... - throw new IOException( - "File " + file.toURI() + " doesn't have a valid URL"); - } - return conf; - } - - /** - * Add a configuration from a file to an existing configuration - * @param conf existing configuration - * @param file file to load - * @param overwrite flag to indicate new values should overwrite the predecessor - * @return the merged configuration - * @throws IOException - */ - public static Configuration addConfigurationFile(Configuration conf, - File file, boolean overwrite) - throws IOException { - Configuration c2 = loadConfFromFile(file, false); - mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite); - return conf; - } - - /** - * Add the system env variables with the given prefix (by convention, env.) - * @param conf existing configuration - * @param prefix prefix - */ - public static void addEnvironmentVariables(Configuration conf, String prefix) { - Map<String, String> env = System.getenv(); - for (Map.Entry<String, String> entry : env.entrySet()) { - conf.set(prefix + entry.getKey(),entry.getValue(), "env"); - } - } - - /** - * looks for the config under $confdir/$templateFilename; if not there - * loads it from /conf/templateFile. - * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the - * origin to help debug what's happening - * @param systemConf system conf - * @param confdir conf dir in FS - * @param templateFilename filename in the confdir - * @param fallbackResource resource to fall back on - * @return loaded conf - * @throws IOException IO problems - */ - public static Configuration loadTemplateConfiguration(Configuration systemConf, - Path confdir, - String templateFilename, - String fallbackResource) throws - IOException { - FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); - - Path templatePath = new Path(confdir, templateFilename); - return loadTemplateConfiguration(fs, templatePath, fallbackResource); - } - - /** - * looks for the config under $confdir/$templateFilename; if not there - * loads it from /conf/templateFile. - * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the - * origin to help debug what's happening. - * @param fs Filesystem - * @param templatePath HDFS path for template - * @param fallbackResource resource to fall back on, or "" for no fallback - * @return loaded conf - * @throws IOException IO problems - * @throws FileNotFoundException if the path doesn't have a file and there - * was no fallback. - */ - public static Configuration loadTemplateConfiguration(FileSystem fs, - Path templatePath, - String fallbackResource) - throws IOException { - Configuration conf; - String origin; - if (fs.exists(templatePath)) { - log.debug("Loading template configuration {}", templatePath); - conf = loadConfiguration(fs, templatePath); - origin = templatePath.toString(); - } else { - if (fallbackResource.isEmpty()) { - throw new FileNotFoundException("No config file found at " + templatePath); - } - log.debug("Template {} not found" + - " -reverting to classpath resource {}", templatePath, fallbackResource); - conf = new Configuration(false); - conf.addResource(fallbackResource); - origin = "Resource " + fallbackResource; - } - //force a get - conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN); - //now set the origin - conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin); - return conf; - } - - - /** - * For testing: dump a configuration - * @param conf configuration - * @return listing in key=value style - */ - public static String dumpConfigToString(Configuration conf) { - Set<String> sorted = sortedConfigKeys(conf); - - StringBuilder builder = new StringBuilder(); - for (String key : sorted) { - - builder.append(key) - .append("=") - .append(conf.get(key)) - .append("\n"); - } - return builder.toString(); - } - - /** - * Merge in one configuration above another - * @param base base config - * @param merge one to merge. This MUST be a non-default-load config to avoid - * merge origin confusion - * @param origin description of the origin for the put operation - * @param overwrite flag to indicate new values should overwrite the predecessor - * @return the base with the merged values - */ - public static Configuration mergeConfigurations(Configuration base, - Iterable<Map.Entry<String, String>> merge, - String origin, - boolean overwrite) { - for (Map.Entry<String, String> entry : merge) { - String key = entry.getKey(); - if (overwrite || base.get(key) == null) { - base.set(key, entry.getValue(), origin); - } - } - return base; - } - - /** - * Register a resource as a default resource. - * Do not attempt to use this unless you understand that the - * order in which default resources are loaded affects the outcome, - * and that subclasses of Configuration often register new default - * resources - * @param resource the resource name - * @return the URL or null - */ - public static URL registerDefaultResource(String resource) { - URL resURL = getResourceUrl(resource); - if (resURL != null) { - Configuration.addDefaultResource(resource); - } - return resURL; - } - - /** - * Load a configuration from a resource on this classpath. - * If the resource is not found, an empty configuration is returned - * @param resource the resource name - * @return the loaded configuration. - */ - public static Configuration loadFromResource(String resource) { - Configuration conf = new Configuration(false); - URL resURL = getResourceUrl(resource); - if (resURL != null) { - log.debug("loaded resources from {}", resURL); - conf.addResource(resource); - } else{ - log.debug("failed to find {} on the classpath", resource); - } - return conf; - - } - - /** - * Get the URL to a resource, null if not on the CP - * @param resource resource to look for - * @return the URL or null - */ - public static URL getResourceUrl(String resource) { - return ConfigHelper.class.getClassLoader() - .getResource(resource); - } - - /** - * Load a resource that must be on the classpath - * @param resource the resource name - * @return the loaded configuration - * @throws FileNotFoundException if the resource is missing - */ - public static Configuration loadMandatoryResource(String resource) - throws FileNotFoundException { - Configuration conf = new Configuration(false); - URL resURL = getResourceUrl(resource); - if (resURL != null) { - log.debug("loaded resources from {}", resURL); - conf.addResource(resource); - } else { - throw new FileNotFoundException(resource); - } - return conf; - } - - /** - * Propagate a property from a source to a dest config, with a best-effort - * attempt at propagating the origin. - * If the - * @param dest destination - * @param src source - * @param key key to try to copy - * @return true if the key was found and propagated - */ - public static boolean propagate(Configuration dest, - Configuration src, - String key) { - String val = src.get(key); - if (val != null) { - String[] origin = src.getPropertySources(key); - if (origin != null && origin.length > 0) { - dest.set(key, val, origin[0]); - } else { - dest.set(key, val); - return true; - } - } - return false; - } - - - /** - * Take a configuration, return a hash map - * @param conf conf - * @return hash map - */ - public static Map<String, String> buildMapFromConfiguration(Configuration conf) { - Map<String, String> map = new HashMap<String, String>(); - return SliderUtils.mergeEntries(map, conf); - } - - /** - * This goes through the keyset of one configuration and retrieves each value - * from a value source -a different or the same configuration. This triggers - * the property resolution process of the value, resolving any variables against - * in-config or inherited configurations - * @param keysource source of keys - * @param valuesource the source of values - * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code> - */ - public static Configuration resolveConfiguration( - Iterable<Map.Entry<String, String>> keysource, - Configuration valuesource) { - Configuration result = new Configuration(false); - for (Map.Entry<String, String> entry : keysource) { - String key = entry.getKey(); - String value = valuesource.get(key); - Preconditions.checkState(value != null, - "no reference for \"%s\" in values", key); - result.set(key, value); - } - return result; - } - - /** - * Register anything we consider deprecated - */ - public static void registerDeprecatedConfigItems() { - Configuration.addDeprecation( - SliderXmlConfKeys.REGISTRY_ZK_QUORUM, - RegistryConstants.KEY_REGISTRY_ZK_QUORUM); - Configuration.addDeprecation( - SliderXmlConfKeys.REGISTRY_PATH, - RegistryConstants.KEY_REGISTRY_ZK_ROOT); - - } - - /** - * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource - * included - * @return a configuration instance - */ - public static Configuration loadSliderConfiguration() { - Configuration conf = new Configuration(); - conf.addResource(SliderKeys.SLIDER_XML); - return conf; - } - - /** - * Inject the {@link SliderKeys#SLIDER_XML} resource - * into the configuration resources <i>of all configurations</i>. - * <p> - * This operation is idempotent. - * <p> - * If the resource is not on the classpath, downgrades, rather than - * fails. - * @return true if the resource was found and loaded. - */ - public static synchronized boolean injectSliderXMLResource() { - if (sliderResourceInjectionAttempted.getAndSet(true)) { - return sliderResourceInjected.get(); - } - URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML); - if (resourceUrl != null) { - Configuration.addDefaultResource(SliderKeys.SLIDER_XML); - sliderResourceInjected.set(true); - } - return sliderResourceInjected.get(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java deleted file mode 100644 index aa5edf1..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ /dev/null @@ -1,915 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import com.google.common.base.Preconditions; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.apache.slider.core.persist.Filenames; -import org.apache.slider.core.persist.InstancePaths; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FilenameFilter; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; -import java.util.Enumeration; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS; -import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS; - -public class CoreFileSystem { - private static final Logger - log = LoggerFactory.getLogger(CoreFileSystem.class); - - private static final String UTF_8 = "UTF-8"; - - protected final FileSystem fileSystem; - protected final Configuration configuration; - - public CoreFileSystem(FileSystem fileSystem, Configuration configuration) { - Preconditions.checkNotNull(fileSystem, - "Cannot create a CoreFileSystem with a null FileSystem"); - Preconditions.checkNotNull(configuration, - "Cannot create a CoreFileSystem with a null Configuration"); - this.fileSystem = fileSystem; - this.configuration = configuration; - } - - public CoreFileSystem(Configuration configuration) throws IOException { - Preconditions.checkNotNull(configuration, - "Cannot create a CoreFileSystem with a null Configuration"); - this.fileSystem = FileSystem.get(configuration); - this.configuration = fileSystem.getConf(); - } - - /** - * Get the temp path for this cluster - * @param clustername name of the cluster - * @return path for temp files (is not purged) - */ - public Path getTempPathForCluster(String clustername) { - Path clusterDir = buildClusterDirPath(clustername); - return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX); - } - - /** - * Returns the underlying FileSystem for this object. - * - * @return filesystem - */ - public FileSystem getFileSystem() { - return fileSystem; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("CoreFileSystem{"); - sb.append("fileSystem=").append(fileSystem.getUri()); - sb.append('}'); - return sb.toString(); - } - - /** - * Build up the path string for a cluster instance -no attempt to - * create the directory is made - * - * @param clustername name of the cluster - * @return the path for persistent data - */ - public Path buildClusterDirPath(String clustername) { - Preconditions.checkNotNull(clustername); - Path path = getBaseApplicationPath(); - return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername); - } - - /** - * Build up the path string for app def folder -no attempt to - * create the directory is made - * - * @param clustername name of the cluster - * @return the path for persistent data - */ - public Path buildAppDefDirPath(String clustername) { - Path path = buildClusterDirPath(clustername); - return new Path(path, SliderKeys.APP_DEF_DIR); - } - - /** - * Build up the path string for addon folder -no attempt to - * create the directory is made - * - * @param clustername name of the cluster - * @return the path for persistent data - */ - public Path buildAddonDirPath(String clustername, String addonId) { - Preconditions.checkNotNull(addonId); - Path path = buildClusterDirPath(clustername); - return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId); - } - - /** - * Build up the path string for package install location -no attempt to - * create the directory is made - * - * @return the path for persistent app package - */ - public Path buildPackageDirPath(String packageName, String packageVersion) { - Preconditions.checkNotNull(packageName); - Path path = getBaseApplicationPath(); - path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName); - if (SliderUtils.isSet(packageVersion)) { - path = new Path(path, packageVersion); - } - return path; - } - - /** - * Build up the path string for package install location -no attempt to - * create the directory is made - * - * @return the path for persistent app package - */ - public Path buildClusterSecurityDirPath(String clusterName) { - Preconditions.checkNotNull(clusterName); - Path path = buildClusterDirPath(clusterName); - return new Path(path, SliderKeys.SECURITY_DIR); - } - - /** - * Build up the path string for keytab install location -no attempt to - * create the directory is made - * - * @return the path for keytab - */ - public Path buildKeytabInstallationDirPath(String keytabFolder) { - Preconditions.checkNotNull(keytabFolder); - Path path = getBaseApplicationPath(); - return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder); - } - - /** - * Build up the path string for keytab install location -no attempt to - * create the directory is made - * - * @return the path for keytab installation location - */ - public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) { - Path homePath = getHomeDirectory(); - Path baseKeytabDir; - if (keytabDir != null) { - baseKeytabDir = new Path(homePath, keytabDir); - } else { - baseKeytabDir = new Path(buildClusterDirPath(clusterName), - SliderKeys.KEYTAB_DIR); - } - return keytabName == null ? baseKeytabDir : - new Path(baseKeytabDir, keytabName); - } - - /** - * Build up the path string for resource install location -no attempt to - * create the directory is made - * - * @return the path for resource - */ - public Path buildResourcePath(String resourceFolder) { - Preconditions.checkNotNull(resourceFolder); - Path path = getBaseApplicationPath(); - return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder); - } - - /** - * Build up the path string for resource install location -no attempt to - * create the directory is made - * - * @return the path for resource - */ - public Path buildResourcePath(String dirName, String fileName) { - Preconditions.checkNotNull(dirName); - Preconditions.checkNotNull(fileName); - Path path = getBaseApplicationPath(); - return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName); - } - - /** - * Build up the path string for cluster resource install location -no - * attempt to create the directory is made - * - * @return the path for resource - */ - public Path buildClusterResourcePath(String clusterName, String component) { - Preconditions.checkNotNull(clusterName); - Path path = buildClusterDirPath(clusterName); - return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component); - } - - /** - * Build up the path string for cluster resource install location -no - * attempt to create the directory is made - * - * @return the path for resource - */ - public Path buildClusterResourcePath(String clusterName) { - Preconditions.checkNotNull(clusterName); - Path path = buildClusterDirPath(clusterName); - return new Path(path, SliderKeys.RESOURCE_DIR); - } - - /** - * Create the Slider cluster path for a named cluster and all its subdirs - * This is a directory; a mkdirs() operation is executed - * to ensure that it is there. - * - * @param clustername name of the cluster - * @return the path to the cluster directory - * @throws java.io.IOException trouble - * @throws SliderException slider-specific exceptions - */ - public Path createClusterDirectories(String clustername, Configuration conf) - throws IOException, SliderException { - - - Path clusterDirectory = buildClusterDirPath(clustername); - InstancePaths instancePaths = new InstancePaths(clusterDirectory); - createClusterDirectories(instancePaths); - return clusterDirectory; - } - - /** - * Create the Slider cluster path for a named cluster and all its subdirs - * This is a directory; a mkdirs() operation is executed - * to ensure that it is there. - * - * @param instancePaths instance paths - * @throws IOException trouble - * @throws SliderException slider-specific exceptions - */ - public void createClusterDirectories(InstancePaths instancePaths) throws - IOException, SliderException { - Path instanceDir = instancePaths.instanceDir; - - verifyDirectoryNonexistent(instanceDir); - FsPermission clusterPerms = getInstanceDirectoryPermissions(); - createWithPermissions(instanceDir, clusterPerms); - createWithPermissions(instancePaths.snapshotConfPath, clusterPerms); - createWithPermissions(instancePaths.generatedConfPath, clusterPerms); - createWithPermissions(instancePaths.historyPath, clusterPerms); - createWithPermissions(instancePaths.tmpPathAM, clusterPerms); - - // Data Directory - String dataOpts = - configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS, - SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS); - log.debug("Setting data directory permissions to {}", dataOpts); - createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts)); - - } - - /** - * Create a directory with the given permissions. - * - * @param dir directory - * @param clusterPerms cluster permissions - * @throws IOException IO problem - * @throws BadClusterStateException any cluster state problem - */ - public void createWithPermissions(Path dir, FsPermission clusterPerms) throws - IOException, - BadClusterStateException { - if (fileSystem.isFile(dir)) { - // HADOOP-9361 shows some filesystems don't correctly fail here - throw new BadClusterStateException( - "Cannot create a directory over a file %s", dir); - } - log.debug("mkdir {} with perms {}", dir, clusterPerms); - //no mask whatoever - fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); - fileSystem.mkdirs(dir, clusterPerms); - //and force set it anyway just to make sure - fileSystem.setPermission(dir, clusterPerms); - } - - /** - * Get the permissions of a path - * - * @param path path to check - * @return the permissions - * @throws IOException any IO problem (including file not found) - */ - public FsPermission getPathPermissions(Path path) throws IOException { - FileStatus status = fileSystem.getFileStatus(path); - return status.getPermission(); - } - - public FsPermission getInstanceDirectoryPermissions() { - String clusterDirPermsOct = - configuration.get(CLUSTER_DIRECTORY_PERMISSIONS, - DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); - return new FsPermission(clusterDirPermsOct); - } - - /** - * Verify that the cluster directory is not present - * - * @param clustername name of the cluster - * @param clusterDirectory actual directory to look for - * @throws IOException trouble with FS - * @throws SliderException If the directory exists - */ - public void verifyClusterDirectoryNonexistent(String clustername, - Path clusterDirectory) - throws IOException, SliderException { - if (fileSystem.exists(clusterDirectory)) { - throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS, - ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername, - clusterDirectory); - } - } - /** - * Verify that the given directory is not present - * - * @param clusterDirectory actual directory to look for - * @throws IOException trouble with FS - * @throws SliderException If the directory exists - */ - public void verifyDirectoryNonexistent(Path clusterDirectory) throws - IOException, - SliderException { - if (fileSystem.exists(clusterDirectory)) { - - log.error("Dir {} exists: {}", - clusterDirectory, - listFSDir(clusterDirectory)); - throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS, - ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS, - clusterDirectory); - } - } - - /** - * Verify that a user has write access to a directory. - * It does this by creating then deleting a temp file - * - * @param dirPath actual directory to look for - * @throws FileNotFoundException file not found - * @throws IOException trouble with FS - * @throws BadClusterStateException if the directory is not writeable - */ - public void verifyDirectoryWriteAccess(Path dirPath) throws IOException, - SliderException { - verifyPathExists(dirPath); - Path tempFile = new Path(dirPath, "tmp-file-for-checks"); - try { - FSDataOutputStream out ; - out = fileSystem.create(tempFile, true); - IOUtils.closeStream(out); - fileSystem.delete(tempFile, false); - } catch (IOException e) { - log.warn("Failed to create file {}: {}", tempFile, e); - throw new BadClusterStateException(e, - "Unable to write to directory %s : %s", dirPath, e.toString()); - } - } - - /** - * Verify that a path exists - * @param path path to check - * @throws FileNotFoundException file not found - * @throws IOException trouble with FS - */ - public void verifyPathExists(Path path) throws IOException { - if (!fileSystem.exists(path)) { - throw new FileNotFoundException(path.toString()); - } - } - - /** - * Verify that a path exists - * @param path path to check - * @throws FileNotFoundException file not found or is not a file - * @throws IOException trouble with FS - */ - public void verifyFileExists(Path path) throws IOException { - FileStatus status = fileSystem.getFileStatus(path); - - if (!status.isFile()) { - throw new FileNotFoundException("Not a file: " + path.toString()); - } - } - - /** - * Given a path, check if it exists and is a file - * - * @param path - * absolute path to the file to check - * @returns true if and only if path exists and is a file, false for all other - * reasons including if file check throws IOException - */ - public boolean isFile(Path path) { - boolean isFile = false; - try { - FileStatus status = fileSystem.getFileStatus(path); - if (status.isFile()) { - isFile = true; - } - } catch (IOException e) { - // ignore, isFile is already set to false - } - return isFile; - } - - /** - * Verify that a file exists in the zip file given by path - * @param path path to zip file - * @param file file expected to be in zip - * @throws FileNotFoundException file not found or is not a zip file - * @throws IOException trouble with FS - */ - public void verifyFileExistsInZip(Path path, String file) throws IOException { - fileSystem.copyToLocalFile(path, new Path("/tmp")); - File dst = new File((new Path("/tmp", path.getName())).toString()); - Enumeration<? extends ZipEntry> entries; - ZipFile zipFile = new ZipFile(dst); - boolean found = false; - - try { - entries = zipFile.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - String nm = entry.getName(); - if (nm.endsWith(file)) { - found = true; - break; - } - } - } finally { - zipFile.close(); - } - dst.delete(); - if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path); - log.info("Verification of " + path + " passed"); - } - /** - * Create the application-instance specific temporary directory - * in the DFS - * - * @param clustername name of the cluster - * @param subdir application ID - * @return the path; this directory will already have been created - */ - public Path createAppInstanceTempPath(String clustername, String subdir) - throws IOException { - Path tmp = getTempPathForCluster(clustername); - Path instancePath = new Path(tmp, subdir); - fileSystem.mkdirs(instancePath); - return instancePath; - } - - /** - * Create the application-instance specific temporary directory - * in the DFS - * - * @param clustername name of the cluster - * @return the path; this directory will already have been deleted - */ - public Path purgeAppInstanceTempFiles(String clustername) throws - IOException { - Path tmp = getTempPathForCluster(clustername); - fileSystem.delete(tmp, true); - return tmp; - } - - /** - * Get the base path - * - * @return the base path optionally configured by - * {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH} - */ - public Path getBaseApplicationPath() { - String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH); - return configuredBasePath != null ? new Path(configuredBasePath) : - new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY); - } - - /** - * Get slider dependency parent dir in HDFS - * - * @return the parent dir path of slider.tar.gz in HDFS - */ - public Path getDependencyPath() { - String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR - + SliderKeys.SLIDER_DEPENDENCY_DIR - : SliderKeys.SLIDER_DEPENDENCY_DIR; - Path dependencyPath = new Path(String.format(parentDir, - SliderUtils.getSliderVersion())); - return dependencyPath; - } - - /** - * Get slider.tar.gz absolute filepath in HDFS - * - * @return the absolute path to slider.tar.gz in HDFS - */ - public Path getDependencyTarGzip() { - Path dependencyLibAmPath = getDependencyPath(); - Path dependencyLibTarGzip = new Path( - dependencyLibAmPath.toUri().toString(), - SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME - + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); - return dependencyLibTarGzip; - } - - public Path getHomeDirectory() { - return fileSystem.getHomeDirectory(); - } - - public boolean maybeAddImagePath(Map<String, LocalResource> localResources, - Path imagePath) throws IOException { - if (imagePath != null) { - LocalResource resource = createAmResource(imagePath, - LocalResourceType.ARCHIVE); - localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource); - return true; - } else { - return false; - } - } - - public boolean maybeAddImagePath(Map<String, LocalResource> localResources, - String imagePath) throws IOException { - - return imagePath != null && - maybeAddImagePath(localResources, new Path(imagePath)); - } - - - - - /** - * Create an AM resource from the - * - * @param destPath dest path in filesystem - * @param resourceType resource type - * @return the resource set up wih application-level visibility and the - * timestamp & size set from the file stats. - */ - public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException { - FileStatus destStatus = fileSystem.getFileStatus(destPath); - LocalResource amResource = Records.newRecord(LocalResource.class); - amResource.setType(resourceType); - // Set visibility of the resource - // Setting to most private option - amResource.setVisibility(LocalResourceVisibility.APPLICATION); - // Set the resource to be copied over - amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem - .resolvePath(destStatus.getPath()))); - // Set timestamp and length of file so that the framework - // can do basic sanity checks for the local resource - // after it has been copied over to ensure it is the same - // resource the client intended to use with the application - amResource.setTimestamp(destStatus.getModificationTime()); - amResource.setSize(destStatus.getLen()); - return amResource; - } - - /** - * Register all files under a fs path as a directory to push out - * - * @param srcDir src dir - * @param destRelativeDir dest dir (no trailing /) - * @return the map of entries - */ - public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException { - //now register each of the files in the directory to be - //copied to the destination - FileStatus[] fileset = fileSystem.listStatus(srcDir); - Map<String, LocalResource> localResources = - new HashMap<String, LocalResource>(fileset.length); - for (FileStatus entry : fileset) { - - LocalResource resource = createAmResource(entry.getPath(), - LocalResourceType.FILE); - String relativePath = destRelativeDir + "/" + entry.getPath().getName(); - localResources.put(relativePath, resource); - } - return localResources; - } - - /** - * Submit a JAR containing a specific class, returning - * the resource to be mapped in - * - * @param clazz class to look for - * @param subdir subdirectory (expected to end in a "/") - * @param jarName <i>At the destination</i> - * @return the local resource ref - * @throws IOException trouble copying to HDFS - */ - public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName) - throws IOException, SliderException { - File localFile = SliderUtils.findContainingJarOrFail(clazz); - return submitFile(localFile, tempPath, subdir, jarName); - } - - /** - * Submit a local file to the filesystem references by the instance's cluster - * filesystem - * - * @param localFile filename - * @param subdir subdirectory (expected to end in a "/") - * @param destFileName destination filename - * @return the local resource ref - * @throws IOException trouble copying to HDFS - */ - public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName) - throws IOException { - Path src = new Path(localFile.toString()); - Path subdirPath = new Path(tempPath, subdir); - fileSystem.mkdirs(subdirPath); - Path destPath = new Path(subdirPath, destFileName); - log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath); - - fileSystem.copyFromLocalFile(false, true, src, destPath); - - // Set the type of resource - file or archive - // archives are untarred at destination - // we don't need the jar file to be untarred for now - return createAmResource(destPath, LocalResourceType.FILE); - } - - /** - * Submit the AM tar.gz resource referenced by the instance's cluster - * filesystem. Also, update the providerResources object with the new - * resource. - * - * @param providerResources - * the provider resource map to be updated - * @throws IOException - * trouble copying to HDFS - */ - public void submitTarGzipAndUpdate( - Map<String, LocalResource> providerResources) throws IOException, - BadClusterStateException { - Path dependencyLibTarGzip = getDependencyTarGzip(); - LocalResource lc = createAmResource(dependencyLibTarGzip, - LocalResourceType.ARCHIVE); - providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc); - } - - /** - * Copy local file(s) to destination HDFS directory. If {@code localPath} is a - * local directory then all files matching the {@code filenameFilter} - * (optional) are copied, otherwise {@code filenameFilter} is ignored. - * - * @param localPath - * a local file or directory path - * @param filenameFilter - * if {@code localPath} is a directory then filenameFilter is used as - * a filter (if specified) - * @param destDir - * the destination HDFS directory where the file(s) should be copied - * @param fp - * file permissions of all the directories and files that will be - * created in this api - * @throws IOException - */ - public void copyLocalFilesToHdfs(File localPath, - FilenameFilter filenameFilter, Path destDir, FsPermission fp) - throws IOException { - if (localPath == null || destDir == null) { - throw new IOException("Either localPath or destDir is null"); - } - fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, - "000"); - fileSystem.mkdirs(destDir, fp); - if (localPath.isDirectory()) { - // copy all local files under localPath to destDir (honoring filename - // filter if provided - File[] localFiles = localPath.listFiles(filenameFilter); - Path[] localFilePaths = new Path[localFiles.length]; - int i = 0; - for (File localFile : localFiles) { - localFilePaths[i++] = new Path(localFile.getPath()); - } - log.info("Copying {} files from {} to {}", i, localPath.toURI(), - destDir.toUri()); - fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir); - } else { - log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri()); - fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), - destDir); - } - // set permissions for all the files created in the destDir - fileSystem.setPermission(destDir, fp); - } - - public void copyLocalFileToHdfs(File localPath, - Path destPath, FsPermission fp) - throws IOException { - if (localPath == null || destPath == null) { - throw new IOException("Either localPath or destPath is null"); - } - fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, - "000"); - fileSystem.mkdirs(destPath.getParent(), fp); - log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri()); - - fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), - destPath); - // set file permissions of the destPath - fileSystem.setPermission(destPath, fp); - } - - public void copyHdfsFileToLocal(Path hdfsPath, File destFile) - throws IOException { - if (hdfsPath == null || destFile == null) { - throw new IOException("Either hdfsPath or destPath is null"); - } - log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI()); - - Path destPath = new Path(destFile.getPath()); - fileSystem.copyToLocalFile(hdfsPath, destPath); - } - - /** - * list entries in a filesystem directory - * - * @param path directory - * @return a listing, one to a line - * @throws IOException - */ - public String listFSDir(Path path) throws IOException { - FileStatus[] stats = fileSystem.listStatus(path); - StringBuilder builder = new StringBuilder(); - for (FileStatus stat : stats) { - builder.append(stat.getPath().toString()) - .append("\t") - .append(stat.getLen()) - .append("\n"); - } - return builder.toString(); - } - - /** - * List all application instances persisted for this user, giving the - * path. The instance name is the last element in the path - * @return a possibly empty map of application instance names to paths - */ - public Map<String, Path> listPersistentInstances() throws IOException { - FileSystem fs = getFileSystem(); - Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY); - log.debug("Looking for all persisted application at {}", path.toString()); - if (!fs.exists(path)) { - // special case: no instances have ever been created - return new HashMap<String, Path>(0); - } - FileStatus[] statuses = fs.listStatus(path); - Map<String, Path> instances = new HashMap<String, Path>(statuses.length); - - // enum the child entries - for (FileStatus status : statuses) { - if (status.isDirectory()) { - // for directories, look for an internal.json underneath - Path child = status.getPath(); - Path internalJson = new Path(child, Filenames.INTERNAL); - if (fs.exists(internalJson)) { - // success => this is an instance - instances.put(child.getName(), child); - } else { - log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.", - child.toString()); - } - } - } - return instances; - } - - public void touch(Path path, boolean overwrite) throws IOException { - FSDataOutputStream out = null; - try { - out = fileSystem.create(path, overwrite); - } finally { - IOUtils.closeStream(out); - } - } - - public void cat(Path path, boolean overwrite, String data) throws IOException { - FSDataOutputStream out = null; - try { - out = fileSystem.create(path, overwrite); - byte[] bytes = data.getBytes(Charset.forName("UTF-8")); - out.write(bytes); - } finally { - IOUtils.closeStream(out); - } - } - - public String cat(Path path) throws IOException { - FileStatus status = fileSystem.getFileStatus(path); - byte[] b = new byte[(int) status.getLen()]; - FSDataInputStream in = null; - try { - in = fileSystem.open(path); - int count = in.read(b); - return new String(b, 0, count, UTF_8); - } finally { - IOUtils.closeStream(in); - } - } - - /** - * Create a path that must exist in the cluster fs - * @param uri uri to create - * @return the path - * @throws SliderException if the path does not exist - */ - public Path createPathThatMustExist(String uri) throws - SliderException, IOException { - Preconditions.checkNotNull(uri); - Path path = new Path(uri); - verifyPathExists(path); - return path; - } - - /** - * Locate an application conf json in the FS. This includes a check to verify - * that the file is there. - * - * @param clustername name of the cluster - * @return the path to the spec. - * @throws IOException IO problems - * @throws SliderException if the path isn't there - */ - public Path locateInstanceDefinition(String clustername) throws IOException, - SliderException { - Path clusterDirectory = buildClusterDirPath(clustername); - Path appConfPath = - new Path(clusterDirectory, Filenames.APPCONF); - verifyClusterSpecExists(clustername, appConfPath); - return appConfPath; - } - - /** - * Verify that a cluster specification exists - * @param clustername name of the cluster (For errors only) - * @param clusterSpecPath cluster specification path - * @throws IOException IO problems - * @throws SliderException if the cluster specification is not present - */ - public void verifyClusterSpecExists(String clustername, Path clusterSpecPath) - throws IOException, - SliderException { - if (!fileSystem.isFile(clusterSpecPath)) { - log.debug("Missing specification file {}", clusterSpecPath); - throw UnknownApplicationInstanceException.unknownInstance( - clustername + "\n (definition not found at " + clusterSpecPath); - } - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java b/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java deleted file mode 100644 index e5fa424..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import java.io.Closeable; - -/** - * A duration in milliseconds. This class can be used - * to count time, and to be polled to see if a time limit has - * passed. - */ -public class Duration implements Closeable { - public long start, finish; - public final long limit; - - /** - * Create a duration instance with a limit of 0 - */ - public Duration() { - this(0); - } - - /** - * Create a duration with a limit specified in millis - * @param limit duration in milliseconds - */ - public Duration(long limit) { - this.limit = limit; - } - - /** - * Start - * @return self - */ - public Duration start() { - start = now(); - return this; - } - - /** - * The close operation relays to {@link #finish()}. - * Implementing it allows Duration instances to be automatically - * finish()'d in Java7 try blocks for when used in measuring durations. - */ - @Override - public final void close() { - finish(); - } - - public void finish() { - finish = now(); - } - - protected long now() { - return System.nanoTime()/1000000; - } - - public long getInterval() { - return finish - start; - } - - /** - * return true if the limit has been exceeded - * @return true if a limit was set and the current time - * exceeds it. - */ - public boolean getLimitExceeded() { - return limit >= 0 && ((now() - start) > limit); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Duration"); - if (finish >= start) { - builder.append(" finished at ").append(getInterval()).append(" millis;"); - } else { - if (start > 0) { - builder.append(" started but not yet finished;"); - } else { - builder.append(" unstarted;"); - } - } - if (limit > 0) { - builder.append(" limit: ").append(limit).append(" millis"); - if (getLimitExceeded()) { - builder.append(" - exceeded"); - } - } - return builder.toString(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java deleted file mode 100644 index 64783b6..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.common.tools; - -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * a scanner which can take an input string for a range or scan the lot. - */ -public class PortScanner { - private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$"); - private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$"); - - private List<Integer> remainingPortsToCheck; - - public PortScanner() { - } - - public void setPortRange(String input) throws BadConfigException { - // first split based on commas - Set<Integer> inputPorts= new TreeSet<Integer>(); - String[] ranges = input.split(","); - for ( String range : ranges ) { - if (range.trim().isEmpty()) { - continue; - } - Matcher m = SINGLE_NUMBER.matcher(range.trim()); - if (m.find()) { - inputPorts.add(Integer.parseInt(m.group())); - continue; - } - m = NUMBER_RANGE.matcher(range.trim()); - if (m.find()) { - String[] boundaryValues = m.group(0).split("-"); - int start = Integer.parseInt(boundaryValues[0].trim()); - int end = Integer.parseInt(boundaryValues[1].trim()); - if (end < start) { - throw new BadConfigException("End of port range is before start: " - + range + " in input: " + input); - } - for (int i = start; i < end + 1; i++) { - inputPorts.add(i); - } - continue; - } - throw new BadConfigException("Bad port range: " + range + " in input: " - + input); - } - if (inputPorts.size() == 0) { - throw new BadConfigException("No ports found in range: " + input); - } - this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts); - } - - public List<Integer> getRemainingPortsToCheck() { - return remainingPortsToCheck; - } - - public int getAvailablePort() throws SliderException, IOException { - if (remainingPortsToCheck != null) { - return getAvailablePortViaPortArray(); - } else { - return SliderUtils.getOpenPort(); - } - } - - private int getAvailablePortViaPortArray() throws SliderException { - boolean found = false; - int availablePort = -1; - Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator(); - while (portsToCheck.hasNext() && !found) { - int portToCheck = portsToCheck.next(); - found = SliderUtils.isPortAvailable(portToCheck); - if (found) { - availablePort = portToCheck; - portsToCheck.remove(); - } - } - - if (availablePort < 0) { - throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION, - "No available ports found in configured range {}", - remainingPortsToCheck); - } - - return availablePort; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java deleted file mode 100644 index 294f37e..0000000 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.tools; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -import java.io.IOException; - -/** - * Extends Core Filesystem with operations to manipulate ClusterDescription - * persistent state - */ -public class SliderFileSystem extends CoreFileSystem { - - public SliderFileSystem(FileSystem fileSystem, - Configuration configuration) { - super(fileSystem, configuration); - } - - public SliderFileSystem(Configuration configuration) throws IOException { - super(configuration); - } - - -}
