IGNITE-2435 Fixed
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6520d4ac Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6520d4ac Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6520d4ac Branch: refs/heads/ignite-2435 Commit: 6520d4ac9a40b19afc8b51a3a14e3e822986086e Parents: 0cd0bac Author: nikolay_tikhonov <[email protected]> Authored: Wed May 4 11:19:25 2016 +0300 Committer: nikolay_tikhonov <[email protected]> Committed: Wed May 4 11:19:25 2016 +0300 ---------------------------------------------------------------------- modules/mesos/pom.xml | 43 ++- .../ignite/mesos/resource/IgniteProvider.java | 13 +- .../ignite/mesos/resource/ResourceProvider.java | 2 +- .../apache/ignite/yarn/ApplicationMaster.java | 21 +- .../apache/ignite/yarn/ClusterProperties.java | 36 ++- .../org/apache/ignite/yarn/IgniteProvider.java | 299 ++++++------------- .../apache/ignite/yarn/IgniteYarnClient.java | 32 +- .../ignite/yarn/utils/IgniteYarnUtils.java | 7 +- 8 files changed, 198 insertions(+), 255 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/mesos/pom.xml ---------------------------------------------------------------------- diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml index fa38265..f2f1acd 100644 --- a/modules/mesos/pom.xml +++ b/modules/mesos/pom.xml @@ -20,7 +20,8 @@ <!-- POM file. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> @@ -36,6 +37,8 @@ <properties> <mesos.version>0.22.0</mesos.version> + <ignite.latest.url>http://ignite.run/download_ignite.php</ignite.latest.url> + <ignite.direct.url>https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.direct.url> </properties> <dependencies> @@ -73,6 +76,44 @@ </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>update-versions</id> + <goals> + <goal>run</goal> + </goals> + <phase>generate-sources</phase> + <configuration> + <target> + <echo message="Update download url in mesos module." /> + <echo message="Direct link ${ignite.direct.url}." /> + <echo message="Latest link ${ignite.latest.url}." /> + + <replaceregexp byline="true" encoding="UTF-8"> + <regexp pattern="(.*DOWNLOAD_LINK = ")(.*)(".*)" /> + <substitution expression="\1${ignite.latest.url}\3"/> + <fileset dir="${basedir}/"> + <include name="**/IgniteProvider.java" /> + </fileset> + </replaceregexp> + + <replaceregexp byline="true" encoding="UTF-8"> + <regexp pattern="(.*DOWNLOAD_URL_PATTERN = ")(.*)(".*)" /> + <substitution expression="\1${ignite.direct.url}\3"/> + <fileset dir="${basedir}/"> + <include name="**/IgniteProvider.java" /> + </fileset> + </replaceregexp> + </target> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> <artifactId>maven-assembly-plugin</artifactId> <version>2.4.1</version> <configuration> http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java ---------------------------------------------------------------------- diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java index f35a426..58079ff 100644 --- a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java +++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java @@ -29,12 +29,12 @@ import org.apache.ignite.mesos.ClusterProperties; * Class downloads and stores Ignite. */ public class IgniteProvider { + // This constants are set by maven-ant-plugin. /** */ - public static final String DOWNLOAD_LINK = "http://www.ignite.run/download_ignite.php";; + private static final String DOWNLOAD_LINK = "http://ignite.run/download_ignite.php";; /** */ - public static final String DOWNLOAD_URL_PATTERN = - "http://www.us.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip";; + private static final String DOWNLOAD_URL_PATTERN = "https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip";; /** */ private String downloadFolder; @@ -61,6 +61,8 @@ public class IgniteProvider { * @return Ignite archive. */ public String downloadIgnite(String ver) throws IOException { + assert ver != null; + URL url; if (ver.equals(ClusterProperties.DEFAULT_IGNITE_VERSION)) { @@ -73,7 +75,8 @@ public class IgniteProvider { if (code == 200) url = conn.getURL(); else - throw new RuntimeException("Failed. Got unexpected response code. Response code: " + code); + throw new RuntimeException("Failed to download ignite distributive. Maybe set incorrect version? " + + "[resCode:" + code + ", ver: " + ver + "]"); } else url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver.replace("-incubating", ""), ver)); @@ -85,7 +88,7 @@ public class IgniteProvider { * Downloads ignite by URL if this version wasn't downloaded before. * * @param url URL to Ignite. - * @return File name in + * @return File name. */ private String downloadIgnite(URL url) { assert url != null; http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java ---------------------------------------------------------------------- diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java index 965e9ad..07bd63b 100644 --- a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java +++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/ResourceProvider.java @@ -72,7 +72,7 @@ public class ResourceProvider { if (provider.fileExist(props.ignitePackagePath())) igniteUrl = baseUrl + IGNITE_PREFIX + props.ignitePackagePath(); else - throw new IllegalArgumentException("Failed. Could not find Ignite by path: " + throw new IllegalArgumentException("Failed to find Ignite by path: " + props.ignitePackagePath()); } http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java index 609f29b..ba63e84 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java @@ -131,9 +131,9 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { resources.put("ignite", IgniteYarnUtils.setupFile(ignitePath, fs, LocalResourceType.ARCHIVE)); resources.put("ignite-config.xml", IgniteYarnUtils.setupFile(cfgPath, fs, LocalResourceType.FILE)); - if (props.licencePath() != null) - resources.put("gridgain-license.xml", - IgniteYarnUtils.setupFile(new Path(props.licencePath()), fs, LocalResourceType.FILE)); + if (props.additionalConfiguration() != null) + resources.put("additional-config.xml", + IgniteYarnUtils.setupFile(new Path(props.additionalConfiguration()), fs, LocalResourceType.FILE)); if (props.userLibs() != null) resources.put("libs", IgniteYarnUtils.setupFile(new Path(props.userLibs()), fs, @@ -143,13 +143,14 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { ctx.setCommands( Collections.singletonList( - (props.licencePath() != null ? "cp gridgain-license.xml ./ignite/*/ || true && " : "") + (props.additionalConfiguration() != null + ? "cp additional-config.xml ./ignite/*/ || true && " : "") + "cp -r ./libs/* ./ignite/*/libs/ || true && " + "./ignite/*/bin/ignite.sh " + "./ignite-config.xml" + " -J-Xmx" + ((int)props.memoryPerNode()) + "m" + " -J-Xms" + ((int)props.memoryPerNode()) + "m" - + IgniteYarnUtils.YARN_LOG_OUT + + IgniteYarnUtils.YARN_CONTAINER_LOG_OUT )); log.log(Level.INFO, "Launching container: {0}.", c.getId()); @@ -189,7 +190,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() || cont.getResource().getMemory() < props.totalMemoryPerNode()) { - log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", + log.log(Level.INFO, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); @@ -219,8 +220,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { } /** {@inheritDoc} */ - public synchronized void onContainersCompleted(List<ContainerStatus> statuses) { - for (ContainerStatus status : statuses) { + public synchronized void onContainersCompleted(List<ContainerStatus> stats) { + for (ContainerStatus status : stats) { containers.remove(status.getContainerId()); log.log(Level.INFO, "Container completed. Container id: {0}. State: {1}.", @@ -367,7 +368,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { rmClient.start(); - if (props.igniteCfg() == null || props.igniteCfg().isEmpty()) { + if (props.igniteConfiguration() == null || props.igniteConfiguration().isEmpty()) { InputStream input = Thread.currentThread().getContextClassLoader() .getResourceAsStream(IgniteYarnUtils.DEFAULT_IGNITE_CONFIG); @@ -383,7 +384,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { IOUtils.closeQuietly(outputStream); } else - cfgPath = new Path(props.igniteCfg()); + cfgPath = new Path(props.igniteConfiguration()); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java index 647aef2..d4249d6 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java @@ -86,9 +86,18 @@ public class ClusterProperties { private double nodeCnt = DEFAULT_IGNITE_NODE_COUNT; /** */ - public static final String IGNITE_URL = "IGNITE_URL"; + public static final String IGNITE_VERSION = "IGNITE_VERSION"; + + /** */ + public static final String DEFAULT_IGNITE_VERSION = "latest"; /** Ignite version. */ + private String igniteVer = DEFAULT_IGNITE_VERSION; + + /** */ + public static final String IGNITE_URL = "IGNITE_URL"; + + /** Ignite URL. */ private String igniteUrl = null; /** */ @@ -107,10 +116,10 @@ public class ClusterProperties { private String ignitePath = null; /** */ - public static final String LICENCE_PATH = "LICENCE_PATH"; + public static final String IGNITE_ADDITIONAL_CONFIG_XML = "IGNITE_ADDITIONAL_CONFIG_XML"; - /** Licence path. */ - private String licencePath = null; + /** Additional configuration file. */ + private String additionalCfg = null; /** */ public static final String IGNITE_JVM_OPTS = "IGNITE_JVM_OPTS"; @@ -197,6 +206,13 @@ public class ClusterProperties { } /** + * @return Ignite version. + */ + public String igniteVersion() { + return igniteVer; + } + + /** * @return Memory overhead for requested memory. */ public double memoryOverHeadPerNode() { @@ -280,15 +296,15 @@ public class ClusterProperties { /** * @return Ignite configuration. */ - public String igniteCfg() { + public String igniteConfiguration() { return igniteCfg; } /** * @return Licence path. */ - public String licencePath() { - return licencePath; + public String additionalConfiguration() { + return additionalCfg; } /** @@ -331,9 +347,10 @@ public class ClusterProperties { Math.max( 0.1 * prop.memPerNode, DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE)); prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT); + prop.igniteVer = getStringProperty(IGNITE_VERSION, props, DEFAULT_IGNITE_VERSION); prop.igniteUrl = getStringProperty(IGNITE_URL, props, null); prop.ignitePath = getStringProperty(IGNITE_PATH, props, null); - prop.licencePath = getStringProperty(LICENCE_PATH, props, null); + prop.additionalCfg = getStringProperty(IGNITE_ADDITIONAL_CONFIG_XML, props, null); prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null); prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR); prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR); @@ -397,9 +414,10 @@ public class ClusterProperties { envs.put(IGNITE_MEMORY_PER_NODE, toEnvVal(memPerNode)); envs.put(IGNITE_NODE_COUNT, toEnvVal(nodeCnt)); + envs.put(IGNITE_VERSION, toEnvVal(igniteVer)); envs.put(IGNITE_URL, toEnvVal(igniteUrl)); envs.put(IGNITE_PATH, toEnvVal(ignitePath)); - envs.put(LICENCE_PATH, toEnvVal(licencePath)); + envs.put(IGNITE_ADDITIONAL_CONFIG_XML, toEnvVal(additionalCfg)); envs.put(IGNITE_JVM_OPTS, toEnvVal(jvmOpts)); envs.put(IGNITE_WORKING_DIR, toEnvVal(igniteWorkDir)); envs.put(IGNITE_LOCAL_WORK_DIR, toEnvVal(igniteLocalWorkDir)); http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java index 158a89a..1ec36e2 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java @@ -18,36 +18,27 @@ package org.apache.ignite.yarn; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; import java.nio.channels.Channels; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.ignite.yarn.utils.IgniteYarnUtils; /** * Downloads and stores Ignite. */ -public class IgniteProvider { +class IgniteProvider { + // This constants are set by maven-ant-plugin. /** */ - public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php";; + private static final String DOWNLOAD_LINK = "http://ignite.run/download_ignite.php";; /** */ - private ClusterProperties props; - - /** */ - private String latestVersion = null; + private static final String DOWNLOAD_URL_PATTERN = "https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip";; /** */ - private boolean hdfs = false; + private ClusterProperties props; /** */ private FileSystem fs; @@ -56,202 +47,90 @@ public class IgniteProvider { * @param props Cluster properties. * @param fs Hadoop file system. */ - public IgniteProvider(ClusterProperties props, FileSystem fs) { + IgniteProvider(ClusterProperties props, FileSystem fs) { this.props = props; this.fs = fs; } /** - * @return Latest ignite version. - */ - public Path getIgnite() throws Exception { - File folder = checkDownloadFolder(); - - if (latestVersion == null) { - List<String> localFiles = findIgnites(folder); - List<String> hdfsFiles = findIgnites(fs, props.igniteReleasesDir()); - - String localLatestVersion = findLatestVersion(localFiles); - String hdfsLatestVersion = findLatestVersion(hdfsFiles); - - if (localLatestVersion != null && hdfsLatestVersion != null) { - if (VersionComparator.INSTANCE.compare(hdfsLatestVersion, localLatestVersion) >= 0) { - latestVersion = hdfsLatestVersion; - - hdfs = true; - } - } - else if (localLatestVersion != null) - latestVersion = localLatestVersion; - else if (hdfsLatestVersion != null) { - latestVersion = hdfsLatestVersion; - - hdfs = true; - } - } - - String newVersion = updateIgnite(latestVersion); - - if (latestVersion != null && newVersion.equals(latestVersion)) { - if (hdfs) - return new Path(formatPath(props.igniteReleasesDir(), latestVersion)); - else - return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion), - formatPath(props.igniteReleasesDir(), latestVersion)); - } - else { - latestVersion = newVersion; - - return IgniteYarnUtils.copyLocalToHdfs(fs, formatPath(props.igniteLocalWorkDir(), latestVersion), - formatPath(props.igniteReleasesDir(), latestVersion)); - } - } - - /** - * @param folder Folder. - * @return Ignite archives. + * @param ver Ignite version. + * @return Ignite. */ - private List<String> findIgnites(File folder) { - String[] files = folder.list(); - - List<String> ignites = new ArrayList<>(); - - if (files != null) { - for (String fileName : files) { - if (fileName.contains("gridgain-community-fabric-") && fileName.endsWith(".zip")) - ignites.add(fileName); - } - } + Path getIgnite(String ver) throws Exception { + assert ver != null; - return ignites; - } - - /** - * @param files Files. - * @return latest ignite version. - */ - private String findLatestVersion(List<String> files) { - String latestVersion = null; + checkDownloadFolder(); - if (!files.isEmpty()) { - if (files.size() == 1) - latestVersion = parseVersion(files.get(0)); - else - latestVersion = parseVersion(Collections.max(files, VersionComparator.INSTANCE)); - } + // Download ignite. + String fileName = downloadIgnite(ver); - return latestVersion; + return copyToHdfs(fileName); } /** - * @param fs File system, - * @param folder Folder. - * @return Ignite archives. + * @param url URL. + * @return Ignite */ - private List<String> findIgnites(FileSystem fs, String folder) { - FileStatus[] fileStatuses = null; - - try { - fileStatuses = fs.listStatus(new Path(folder)); - } - catch (FileNotFoundException e) { - // Ignore. Folder doesn't exist. - } - catch (Exception e) { - throw new RuntimeException("Couldnt get list files from hdfs.", e); - } - - List<String> ignites = new ArrayList<>(); + Path getIgnite(URL url) throws Exception { + assert url != null; - if (fileStatuses != null) { - for (FileStatus file : fileStatuses) { - String fileName = file.getPath().getName(); + checkDownloadFolder(); - if (fileName.contains("gridgain-community-fabric-") && fileName.endsWith(".zip")) - ignites.add(fileName); - } - } + // Download ignite. + String fileName = downloadIgnite(url); - return ignites; + return copyToHdfs(fileName); } /** - * @param version Ignite version. - * @return Ignite. + * @param fileName File name. + * @return Path. + * @throws IOException If failed. */ - public Path getIgnite(String version) throws Exception { - checkDownloadFolder(); - - // Download ignite. - String fileName = downloadIgnite(version); - - Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName); + private Path copyToHdfs(String fileName) throws IOException { + Path dst = getHdfsPath(fileName); if (!fs.exists(dst)) - fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst); + fs.copyFromLocalFile(new Path(getLocalPath(fileName)), dst); return dst; } /** - * @param folder folder - * @param version version - * @return Path + * @param ver Ignite version. + * @return File name. + * @throws Exception If failed. */ - private static String formatPath(String folder, String version) { - return folder + File.separator + "gridgain-community-fabric-" + version + ".zip"; - } + private String downloadIgnite(String ver) throws Exception { + assert ver != null; - /** - * @param currentVersion The current latest version. - * @return Current version if the current version is latest; new ignite version otherwise. - */ - private String updateIgnite(String currentVersion) { - try { - URL url; + URL url; - if (currentVersion == null) - url = new URL(DOWNLOAD_LINK); - else - url = new URL(DOWNLOAD_LINK + "?version=" + currentVersion); + if (ver.equals(ClusterProperties.DEFAULT_IGNITE_VERSION)) { + URL updateUrl = new URL(DOWNLOAD_LINK); - HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + HttpURLConnection conn = (HttpURLConnection)updateUrl.openConnection(); int code = conn.getResponseCode(); - if (code == 200) { - String redirectUrl = conn.getURL().toString(); - - checkDownloadFolder(); - - FileOutputStream outFile = new FileOutputStream(props.igniteLocalWorkDir() + File.separator - + fileName(redirectUrl)); - - outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE); - - outFile.close(); - - return parseVersion(redirectUrl); - } - else if (code == 304) - // This version is latest. - return currentVersion; + if (code == 200) + url = conn.getURL(); else - throw new RuntimeException("Got unexpected response code. Response code: " + code); - } - catch (IOException e) { - throw new RuntimeException("Failed update ignite.", e); + throw new RuntimeException("Failed to download ignite distributive. Maybe set incorrect version? " + + "[resCode:" + code + ", ver: " + ver + "]"); } + else + url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver.replace("-incubating", ""), ver)); + + return downloadIgnite(url); } /** - * @param igniteUrl Url to ignite. + * @param url URL to ignite. * @return Ignite file name. */ - private String downloadIgnite(String igniteUrl) { + private String downloadIgnite(URL url) { try { - URL url = new URL(igniteUrl); - HttpURLConnection conn = (HttpURLConnection)url.openConnection(); int code = conn.getResponseCode(); @@ -259,12 +138,10 @@ public class IgniteProvider { if (code == 200) { String fileName = fileName(url.toString()); - String filePath = props.igniteLocalWorkDir() + File.separator + fileName; - - if (new File(filePath).exists()) + if (igniteExist(fileName)) return fileName; - FileOutputStream outFile = new FileOutputStream(filePath); + FileOutputStream outFile = new FileOutputStream(getLocalPath(fileName)); outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE); @@ -273,7 +150,8 @@ public class IgniteProvider { return fileName; } else - throw new RuntimeException("Got unexpected response code. Response code: " + code); + throw new RuntimeException("Failed to download. Got unexpected response code [respCode: " + code + + ", URL: " + url + "]"); } catch (IOException e) { throw new RuntimeException("Failed update ignite.", e); @@ -281,67 +159,58 @@ public class IgniteProvider { } /** - * @return Download folder. + * @return {@code True} if Ignite exist on local disk or hdfs otherwise {@code false}. + * @throws IOException */ - private File checkDownloadFolder() { - File file = new File(props.igniteLocalWorkDir()); - - if (!file.exists()) - file.mkdirs(); - - if (!file.exists()) - throw new RuntimeException("Couldn't create local directory! Path: " + file.toURI()); + private boolean igniteExist(String fileName) throws IOException { + String localPath = getLocalPath(fileName); + Path hdfsPath = getHdfsPath(fileName); - return file; + // Check on local disk and hdfs. + return new File(localPath).exists() || fs.exists(hdfsPath); } /** - * @param url URL. - * @return Ignite version. + * @param fileName File name. + * @return HDFS path. */ - private static String parseVersion(String url) { - String[] split = url.split("-"); - - return split[split.length - 1].replaceAll(".zip", ""); + private Path getHdfsPath(String fileName) { + return new Path(props.igniteReleasesDir() + File.separator + fileName); } /** - * @param url URL. - * @return File name. + * @param fileName File name. + * @return Path on local file system. */ - private static String fileName(String url) { - String[] split = url.split("/"); - - return split[split.length - 1]; + private String getLocalPath(String fileName) { + return props.igniteLocalWorkDir() + File.separator + fileName; } /** - * Ignite version comparator. + * @return Download folder. */ - public static final class VersionComparator implements Comparator<String> { - /** */ - public static final VersionComparator INSTANCE = new VersionComparator(); + private File checkDownloadFolder() { + File file = new File(props.igniteLocalWorkDir()); - /** */ - private VersionComparator() { - // No-op. + if (!file.exists()) { + boolean res = file.mkdirs(); + + assert res; } - /** {@inheritDoc} */ - @Override public int compare(String f1, String f2) { - if (f1.equals(f2)) - return 0; + if (!file.exists()) + throw new RuntimeException("Failed to create local directory. Path: " + file.toURI()); - String[] ver1 = parseVersion(f1).split("\\."); - String[] ver2 = parseVersion(f2).split("\\."); + return file; + } - if (Integer.valueOf(ver1[0]) >= Integer.valueOf(ver2[0]) - && Integer.valueOf(ver1[1]) >= Integer.valueOf(ver2[1]) - && Integer.valueOf(ver1[2]) >= Integer.valueOf(ver2[2])) + /** + * @param url URL. + * @return File name. + */ + private static String fileName(String url) { + String[] split = url.split("/"); - return 1; - else - return -1; - } + return split[split.length - 1]; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java index 2a9a53e..48865d3 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java @@ -19,6 +19,7 @@ package org.apache.ignite.yarn; import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -52,7 +53,7 @@ import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment; */ public class IgniteYarnClient { /** */ - public static final Logger log = Logger.getLogger(IgniteYarnClient.class.getSimpleName()); + private static final Logger log = Logger.getLogger(IgniteYarnClient.class.getSimpleName()); /** * Main methods has one mandatory parameter and one optional parameter. @@ -77,13 +78,18 @@ public class IgniteYarnClient { FileSystem fs = FileSystem.get(conf); - Path ignite; + Path ignitePath; - // Load ignite and jar + // Load ignite and jar. if (props.ignitePath() == null) - ignite = getIgnite(props, fs); - else - ignite = new Path(props.ignitePath()); + ignitePath = getIgnite(props, fs); + else { + ignitePath = new Path(props.ignitePath()); + + if (!fs.exists(ignitePath)) + throw new IllegalArgumentException("Incorrect path to Ignite. " + + "IGNITE_PATH property should contains reachable path in HDFS. IGNITE_PATH: " + props.ignitePath()); + } // Upload the jar file to HDFS. Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar, @@ -95,8 +101,8 @@ public class IgniteYarnClient { amContainer.setCommands( Collections.singletonList( Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName() - + IgniteYarnUtils.SPACE + ignite.toUri() - + IgniteYarnUtils.YARN_LOG_OUT + + IgniteYarnUtils.SPACE + ignitePath.toUri() + + IgniteYarnUtils.YARN_APP_LOG_OUT ) ); @@ -183,17 +189,17 @@ public class IgniteYarnClient { /** * @param props Properties. - * @param fileSystem Hdfs file system. + * @param fs Hdfs file system. * @return Hdfs path to ignite node. * @throws Exception */ - private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception { - IgniteProvider provider = new IgniteProvider(props, fileSystem); + private static Path getIgnite(ClusterProperties props, FileSystem fs) throws Exception { + IgniteProvider provider = new IgniteProvider(props, fs); if (props.igniteUrl() == null) - return provider.getIgnite(); + return provider.getIgnite(props.igniteVersion()); else - return provider.getIgnite(props.igniteUrl()); + return provider.getIgnite(new URL(props.igniteUrl())); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/6520d4ac/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java index 92507a7..e145d4f 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java @@ -46,10 +46,15 @@ public class IgniteYarnUtils { public static final String JAR_NAME = "ignite-yarn.jar"; /** */ - public static final String YARN_LOG_OUT = + public static final String YARN_CONTAINER_LOG_OUT = " 1>" + LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + LOG_DIR_EXPANSION_VAR + "/stderr"; + /** */ + public static final String YARN_APP_LOG_OUT = + " 1>" + LOG_DIR_EXPANSION_VAR + "/stderr" + + " 2>" + LOG_DIR_EXPANSION_VAR + "/stdout"; + /** * @param file Path. * @param fs File system.
