TAVERNA-893 USe HTTPClient 4 and MessageDigest Now supports all Java-supported message digests, including SHA-1 and SHA-256
Local files supported via java.nio Path URIs. No longer supported are classical URLs registered with URLConnection, like ftp:// Project: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/commit/8a82204a Tree: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/tree/8a82204a Diff: http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/diff/8a82204a Branch: refs/heads/master Commit: 8a82204a930e99c587f5d60c5df2318a2d8290fe Parents: d6c43d6 Author: Stian Soiland-Reyes <[email protected]> Authored: Fri Feb 12 17:47:47 2016 +0000 Committer: Stian Soiland-Reyes <[email protected]> Committed: Fri Feb 12 17:47:47 2016 +0000 ---------------------------------------------------------------------- taverna-download-impl/pom.xml | 23 ++- .../download/impl/DownloadManagerImpl.java | 156 +++++++++++++------ .../download/impl/TestDownloadManagerImpl.java | 30 +++- 3 files changed, 147 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/pom.xml ---------------------------------------------------------------------- diff --git a/taverna-download-impl/pom.xml b/taverna-download-impl/pom.xml index bc50c70..e81a1aa 100644 --- a/taverna-download-impl/pom.xml +++ b/taverna-download-impl/pom.xml @@ -16,7 +16,8 @@ limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.taverna.osgi</groupId> @@ -45,21 +46,33 @@ <artifactId>taverna-download-api</artifactId> <version>${project.parent.version}</version> </dependency> - <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>${log4j.version}</version> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>${commons.io.version}</version> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient-osgi</artifactId> + <version>${apache.httpclient.version}</version> + </dependency> + <dependency> + <!-- Needed by httpclient-osgi --> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore-osgi</artifactId> + <version>${apache.httpcore.version}</version> + </dependency> + <dependency> + <!-- Needed by httpclient-osgi --> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.2</version> </dependency> <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> <version>${commons.codec.version}</version> </dependency> + </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java ---------------------------------------------------------------------- diff --git a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java b/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java index 7292c8e..fa76723 100644 --- a/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java +++ b/taverna-download-impl/src/main/java/org/apache/taverna/download/impl/DownloadManagerImpl.java @@ -16,14 +16,25 @@ */ package org.apache.taverna.download.impl; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; +import org.apache.http.client.fluent.Request; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.VersionInfo; import org.apache.log4j.Logger; import org.apache.taverna.download.DownloadException; import org.apache.taverna.download.DownloadManager; @@ -33,81 +44,128 @@ import org.apache.taverna.download.DownloadManager; */ public class DownloadManagerImpl implements DownloadManager { + CloseableHttpClient httpclient = HttpClients.createDefault(); + private static final int TIMEOUT = Integer.getInteger("taverna.download.timeout.seconds", 30) * 1000; - + private static final Logger logger = Logger.getLogger(DownloadManagerImpl.class); - public void download(URL source, File destination) throws DownloadException { + @Override + public void download(URI source, Path destination) throws DownloadException { download(source, destination, null); } - public void download(URL source, File destination, String digestAlgorithm) throws DownloadException { - // TODO Use MessageDigest when Java 7 available - if (digestAlgorithm != null && !digestAlgorithm.equals("MD5")) { - throw new IllegalArgumentException("Only MD5 supported"); - } - URL digestSource = null; + @Override + public void download(URI source, Path destination, String digestAlgorithm) throws DownloadException { + URI digestSource = null; if (digestAlgorithm != null) { - try { - digestSource = new URL(source.toString() + mapAlgorithmToFileExtension(digestAlgorithm)); - } catch (MalformedURLException e) { - throw new DownloadException("Error creating digest URL", e); - } + // Note: Will break with ?download=file.xml kind of URLs + digestSource = source.resolve(source.getPath() + mapAlgorithmToFileExtension(digestAlgorithm)); } download(source, destination, digestAlgorithm, digestSource); } - public void download(URL source, File destination, String digestAlgorithm, URL digestSource) + public String getUserAgent() { + Package pack = getClass().getPackage(); + String httpClientVersion = VersionInfo.getUserAgent("Apache-HttpClient", "org.apache.http.client", + Request.class); + return "Apache-Taverna-OSGi" + "/" + pack.getImplementationVersion() + " (" + httpClientVersion + ")"; + } + + @Override + public void download(URI source, Path destination, String digestAlgorithm, URI digestSource) throws DownloadException { - // TODO Use MessageDigest when Java 7 available - if (digestAlgorithm != null && !digestAlgorithm.equals("MD5")) { - throw new IllegalArgumentException("Only MD5 supported"); + + MessageDigest md = null; + if (digestAlgorithm != null) { + try { + md = MessageDigest.getInstance(digestAlgorithm); + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException("Unsupported digestAlgorithm: " + digestAlgorithm, e); + } } + // download the file - File tempFile; + Path tempFile; try { - tempFile = File.createTempFile("DownloadManager", "tmp"); - tempFile.deleteOnExit(); - logger.info(String.format("Downloading %1$s to %2$s", source, tempFile)); - FileUtils.copyURLToFile(source, tempFile, TIMEOUT, TIMEOUT); - } catch (IOException e) { - throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e); + tempFile = Files.createTempFile(destination.getParent(), "." + destination.getFileName(), ".tmp"); + } catch (IOException e1) { + // perhaps a permission problem? + throw new DownloadException("Can't create temporary file in folder " + destination.getParent(), e1); } + logger.info(String.format("Downloading %1$s to %2$s", source, tempFile)); + downloadToFile(source, tempFile); + if (digestSource != null) { // download the digest file - File digestFile; + String expectedDigest; + expectedDigest = downloadHash(digestSource).trim().toLowerCase(Locale.ROOT); + // check if the digest matches try { - digestFile = File.createTempFile("DownloadManager", "tmp"); - digestFile.deleteOnExit(); - logger.info(String.format("Downloading %1$s to %2$s", digestSource, digestFile)); - FileUtils.copyURLToFile(digestSource, digestFile, TIMEOUT, TIMEOUT); - } catch (IOException e) { - throw new DownloadException(String.format("Error checking digest for %1$s.", source), e); - } - // check the digest matches - try { - String digestString1 = DigestUtils.md5Hex(new FileInputStream(tempFile)); - String digestString2 = FileUtils.readFileToString(digestFile); - if (!digestString1.equals(digestString2)) { - throw new DownloadException(String.format( - "Error downloading file: digsests not equal. (%1$s != %2$s)", - digestString1, digestString2)); + try (InputStream s = Files.newInputStream(tempFile)) { + DigestUtils.updateDigest(md, s); + String actualDigest = Hex.encodeHexString(md.digest()); + if (!actualDigest.equals(expectedDigest)) { + throw new DownloadException( + String.format("Error downloading file: checksum mismatch (%1$s != %2$s)", + actualDigest, expectedDigest)); + } } } catch (IOException e) { - throw new DownloadException(String.format("Error checking digest for %1$s", destination), - e); + throw new DownloadException(String.format("Error checking digest for %1$s", destination), e); } } - // copy file to destination + // All fine, move to destination try { logger.info(String.format("Copying %1$s to %2$s", tempFile, destination)); - FileUtils.copyFile(tempFile, destination); + Files.move(tempFile, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e); } } + private String downloadHash(URI source) throws DownloadException { + try { + // We want to handle http/https with HTTPClient + if (source.getScheme().equalsIgnoreCase("http") || source.getScheme().equalsIgnoreCase("https")) { + logger.info("Downloading checksum " + source); + return Request.Get(source).userAgent(getUserAgent()).connectTimeout(TIMEOUT).socketTimeout(TIMEOUT).execute() + .returnContent().asString(StandardCharsets.ISO_8859_1); + } else { + // Try as a supported Path, e.g. file: or relative path + try { + Path path = Paths.get(source); + return Files.readAllLines(path, StandardCharsets.ISO_8859_1).get(0); + } catch (FileSystemNotFoundException e) { + throw new DownloadException("Unsupported URL scheme: " + source.getScheme()); + } + } + } catch (IOException e) { + throw new DownloadException(String.format("Error downloading %1$s", source), e); + } + } + + private void downloadToFile(URI source, Path destination) throws DownloadException { + try { + // We want to handle http/https with HTTPClient + if (source.getScheme().equalsIgnoreCase("http") || source.getScheme().equalsIgnoreCase("https")) { + Request.Get(source).userAgent(getUserAgent()).connectTimeout(TIMEOUT).socketTimeout(TIMEOUT).execute() + .saveContent(destination.toFile()); + } else { + // Try as a supported Path, e.g. file: or relative path + try { + Path path = Paths.get(source); + Files.copy(path, destination, StandardCopyOption.REPLACE_EXISTING); + } catch (FileSystemNotFoundException e) { + throw new DownloadException("Unsupported URL scheme: " + source.getScheme()); + } + } + } catch (IOException e) { + throw new DownloadException(String.format("Error downloading %1$s to %2$s.", source, destination), e); + } + } + private String mapAlgorithmToFileExtension(String algorithm) { return "." + algorithm.toLowerCase().replaceAll("-", ""); } http://git-wip-us.apache.org/repos/asf/incubator-taverna-osgi/blob/8a82204a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java ---------------------------------------------------------------------- diff --git a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java b/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java index a260c59..fbe2dcb 100644 --- a/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java +++ b/taverna-download-impl/src/test/java/org/apache/taverna/download/impl/TestDownloadManagerImpl.java @@ -3,7 +3,7 @@ package org.apache.taverna.download.impl; import static java.nio.charset.StandardCharsets.US_ASCII; import static org.junit.Assert.assertEquals; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -15,7 +15,7 @@ public class TestDownloadManagerImpl { /** * This test should remain @Ignored - * as it relies on a third-party web site + * as it relies on a web site * and should not break the build. * * Verifies TAVERNA-893 @@ -28,13 +28,13 @@ public class TestDownloadManagerImpl { Path pomFile = Files.createTempFile("test", ".pom"); // NOTE: The below URL is a Taverna 2 POM - not related to // taverna-plugin-impl - URL wrongURL = new URL("http://192.185.115.65/~diana/DIANA_plugin_updated/test-plugins/gr/dianatools/diana.services-activity/1.0-SNAPSHOT/diana.services-activity-1.0-SNAPSHOT.pom";); + URI wrongURL = URI.create("http://192.185.115.65/~diana/DIANA_plugin_updated/test-plugins/gr/dianatools/diana.services-activity/1.0-SNAPSHOT/diana.services-activity-1.0-SNAPSHOT.pom";); // With this we get an exception - but only because the // 300 erorr page fails the MD5 check //dl.download(wrongURL, pomFile.toFile(), "MD5"); // so we'll try without hashsum - dl.download(wrongURL, pomFile.toFile()); + dl.download(wrongURL, pomFile); } @@ -47,7 +47,7 @@ public class TestDownloadManagerImpl { */ @Test public void downloadLocalFile() throws Exception { - Path example = Files.createTempFile("example", ".txt"); + Path example = Files.createTempFile("test", ".txt"); Files.write(example, "Hello world".getBytes(US_ASCII)); // No newline Path exampleMd5 = example.resolveSibling(example.getFileName() + ".md5"); // stain@biggie:~$ echo -n "Hello world"|md5sum @@ -58,11 +58,25 @@ public class TestDownloadManagerImpl { Path toFile = Files.createTempFile("downloaded", ".txt"); - dl.download(example.toUri().toURL(), toFile.toFile(), "MD5"); + dl.download(example.toUri(), toFile, "MD5"); String hello = Files.readAllLines(toFile, US_ASCII).get(0); - assertEquals("Hello world", hello); - + assertEquals("Hello world", hello); + } + + /** + * This test should remain @Ignored + * as it relies on a web site + * and should not break the build. + * + */ + @Ignore + @Test + public void downloadPomSha1() throws Exception { + Path destination = Files.createTempFile("test", "pom"); + URI source = URI.create("https://repo.maven.apache.org/maven2/org/apache/apache/17/apache-17.pom";); + DownloadManagerImpl dl = new DownloadManagerImpl(); + dl.download(source, destination, "SHA-1"); } }
