Repository: ignite Updated Branches: refs/heads/master 69ac976e6 -> 09a823cf6
IGNITE-9388: mesos IgniteProvider tries to access obolete ignite.run or download from slow archive. - Fixes #4639. Signed-off-by: shroman <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/09a823cf Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/09a823cf Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/09a823cf Branch: refs/heads/master Commit: 09a823cf6c435372f75d53f5ddb9d15ce0c3e55b Parents: 69ac976 Author: shroman <[email protected]> Authored: Wed Sep 12 10:10:56 2018 +0900 Committer: shroman <[email protected]> Committed: Wed Sep 12 10:10:56 2018 +0900 ---------------------------------------------------------------------- modules/mesos/pom.xml | 36 ++++-- .../ignite/mesos/resource/IgniteProvider.java | 120 ++++++++++++++++--- 2 files changed, 127 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/09a823cf/modules/mesos/pom.xml ---------------------------------------------------------------------- diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml index a8c7d8b..a5c711a 100644 --- a/modules/mesos/pom.xml +++ b/modules/mesos/pom.xml @@ -36,8 +36,9 @@ <properties> <mesos.version>1.5.0</mesos.version> - <ignite.latest.url>http://ignite.run/download_ignite.php</ignite.latest.url> - <ignite.direct.url>https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.direct.url> + <ignite.version.url>https://ignite.apache.org/latest</ignite.version.url> + <ignite.path>/ignite/%s/apache-ignite-fabric-%s-bin.zip</ignite.path> + <apache.mirror.url>https://www.apache.org/dyn/closer.cgi?as_json=1</apache.mirror.url> </properties> <dependencies> @@ -54,6 +55,12 @@ </dependency> <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.11</version> @@ -76,23 +83,30 @@ <phase>generate-sources</phase> <configuration> <target> - <echo message="Update download url in mesos module." /> - <echo message="Direct link ${ignite.direct.url}." /> - <echo message="Latest link ${ignite.latest.url}." /> + <echo message="Update download url in mesos module."/> + <echo message="Latest version ${ignite.version.url}."/> + + <replaceregexp byline="true" encoding="UTF-8"> + <regexp pattern="(.*IGNITE_LATEST_VERSION_URL = ")(.*)(".*)"/> + <substitution expression="\1${ignite.version.url}\3"/> + <fileset dir="${basedir}/"> + <include name="**/IgniteProvider.java"/> + </fileset> + </replaceregexp> <replaceregexp byline="true" encoding="UTF-8"> - <regexp pattern="(.*DOWNLOAD_LINK = ")(.*)(".*)" /> - <substitution expression="\1${ignite.latest.url}\3" /> + <regexp pattern="(.*APACHE_MIRROR_URL = ")(.*)(".*)"/> + <substitution expression="\1${apache.mirror.url}\3"/> <fileset dir="${basedir}/"> - <include name="**/IgniteProvider.java" /> + <include name="**/IgniteProvider.java"/> </fileset> </replaceregexp> <replaceregexp byline="true" encoding="UTF-8"> - <regexp pattern="(.*DOWNLOAD_URL_PATTERN = ")(.*)(".*)" /> - <substitution expression="\1${ignite.direct.url}\3" /> + <regexp pattern="(.*IGNITE_PATH = ")(.*)(".*)"/> + <substitution expression="\1${ignite.path}\3"/> <fileset dir="${basedir}/"> - <include name="**/IgniteProvider.java" /> + <include name="**/IgniteProvider.java"/> </fileset> </replaceregexp> </target> http://git-wip-us.apache.org/repos/asf/ignite/blob/09a823cf/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java ---------------------------------------------------------------------- diff --git a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java index 3d794db..5735051 100644 --- a/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java +++ b/modules/mesos/src/main/java/org/apache/ignite/mesos/resource/IgniteProvider.java @@ -17,9 +17,13 @@ package org.apache.ignite.mesos.resource; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.nio.channels.Channels; @@ -27,19 +31,38 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.ignite.mesos.ClusterProperties; +import static org.apache.ignite.mesos.ClusterProperties.IGNITE_VERSION; + /** * Class downloads and stores Ignite. */ public class IgniteProvider { - // This constants are set by maven-ant-plugin. - /** */ - private static final String DOWNLOAD_LINK = "http://ignite.run/download_ignite.php";; + /** Logger. */ + private static final Logger log = Logger.getLogger(IgniteProvider.class.getSimpleName()); + // This constants are set by maven-ant-plugin. /** */ private static final String DOWNLOAD_URL_PATTERN = "https://archive.apache.org/dist/ignite/%s/apache-ignite-fabric-%s-bin.zip";; + /** URL for request Ignite latest version. */ + private final static String IGNITE_LATEST_VERSION_URL = ""; + + /** Mirrors. */ + private static final String APACHE_MIRROR_URL = ""; + + /** Ignite on Apache URL path. */ + private static final String IGNITE_PATH = ""; + + /** Version pattern. */ + private static final Pattern VERSION_PATTERN = Pattern.compile("(?<=version=).*\\S+"); + /** */ private String downloadFolder; @@ -52,8 +75,8 @@ public class IgniteProvider { /** * @param ver Ignite version. - * @throws IOException If downloading failed. * @return Path to latest ignite. + * @throws IOException If downloading failed. */ public String getIgnite(String ver) throws IOException { return downloadIgnite(ver); @@ -61,31 +84,90 @@ public class IgniteProvider { /** * @param ver Ignite version which will be downloaded. If {@code null} will download the latest ignite version. - * @throws IOException If downloading failed. * @return Ignite archive. + * @throws IOException If downloading failed. */ public String downloadIgnite(String ver) throws IOException { assert ver != null; URL url; + // get the latest version. if (ver.equals(ClusterProperties.DEFAULT_IGNITE_VERSION)) { - URL updateUrl = new URL(DOWNLOAD_LINK); + try { + ver = findLatestVersion(); - HttpURLConnection conn = (HttpURLConnection)updateUrl.openConnection(); + // and try to retrieve from a mirror. + url = new URL(String.format(findMirror() + IGNITE_PATH, ver, ver)); + } + catch (Exception e) { + // fallback to archive. + url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver, ver)); + } + } + else { + // or from archive. + url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver, ver)); + } - int code = conn.getResponseCode(); + return downloadIgnite(url); + } - if (code == 200) - url = conn.getURL(); - else - throw new RuntimeException("Failed to download ignite distributive. Maybe set incorrect version? " + - "[resCode:" + code + ", ver: " + ver + "]"); - } + /** + * Attempts to retrieve the preferred mirror. + * + * @return Mirror url. + * @throws IOException If failed. + */ + private String findMirror() throws IOException { + String response = getHttpContents(new URL(APACHE_MIRROR_URL)); + + if (response == null) + throw new RuntimeException("Failed to retrieve mirrors"); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode mirrorUrl = mapper.readTree(response).get("preferred"); + + if (mirrorUrl == null) + throw new RuntimeException("Failed to find the preferred mirror"); + + return mirrorUrl.asText(); + } + + /** + * Attempts to obtain the latest version. + * + * @return Latest version. + * @throws IOException If failed. + */ + private String findLatestVersion() throws IOException { + String response = getHttpContents(new URL(IGNITE_LATEST_VERSION_URL)); + + if (response == null) + throw new RuntimeException("Failed to identify the latest version. Specify it with " + IGNITE_VERSION); + + Matcher m = VERSION_PATTERN.matcher(response); + if (m.find()) + return m.group(); else - url = new URL(String.format(DOWNLOAD_URL_PATTERN, ver.replace("-incubating", ""), ver)); + throw new RuntimeException("Failed to retrieve the latest version. Specify it with " + IGNITE_VERSION); + } - return downloadIgnite(url); + /** + * @param url Url. + * @return Contents. + * @throws IOException If failed. + */ + private String getHttpContents(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + + int code = conn.getResponseCode(); + + if (code != 200) + throw null; + + BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8")); + return rd.lines().collect(Collectors.joining()); } /** @@ -110,7 +192,9 @@ public class IgniteProvider { if (fileExist(fileName)) return fileName; - FileOutputStream outFile = new FileOutputStream(downloadFolder + fileName); + log.log(Level.INFO, "Downloading from {0}", url.toString()); + + FileOutputStream outFile = new FileOutputStream(Paths.get(downloadFolder, fileName).toFile()); outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE); @@ -119,7 +203,7 @@ public class IgniteProvider { return fileName; } else - throw new RuntimeException("Got unexpected response code. Response code: " + code); + throw new RuntimeException("Got unexpected response code. Response code: " + code + " from " + url); } catch (IOException e) { throw new RuntimeException("Failed to download Ignite.", e);
