Author: maartenc
Date: Sat Jun 5 21:25:26 2010
New Revision: 951772
URL: http://svn.apache.org/viewvc?rev=951772&view=rev
Log:
IMPROVEMENT: added support for 'deflate' HTTP Content-Encoding (IVY-1194)
Modified:
ant/ivy/core/trunk/CHANGES.txt
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java
Modified: ant/ivy/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/CHANGES.txt?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/CHANGES.txt (original)
+++ ant/ivy/core/trunk/CHANGES.txt Sat Jun 5 21:25:26 2010
@@ -115,7 +115,7 @@ for detailed view of each issue, please
- NEW: Support ivy.xml parent mechanism (IVY-742) (thanks to Jason Trump and
Jean-Louis Boudart)
- NEW: Make ivy.xml <conf description> available (IVY-1158)
-- IMPROVEMENT: added support for 'gzip' HTTP Content-Encoding (IVY-1194)
+- IMPROVEMENT: added support for 'gzip' and 'deflate' HTTP Content-Encoding
(IVY-1194)
- IMPROVEMENT: retrieve doesn't retrive files if the current one is more
recent (IVY-1044)
- IMPROVEMENT: better diagnostics when reporting bad ivy file in cache
(IVY-1190)
- IMPROVEMENT: Support changing="true" for inline retrieve (IVY-1180)
Modified:
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
(original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
Sat Jun 5 21:25:26 2010
@@ -17,13 +17,19 @@
*/
package org.apache.ivy.util.url;
+import java.io.BufferedInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.regex.Pattern;
+import java.util.zip.DataFormatException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
public abstract class AbstractURLHandler implements URLHandler {
@@ -118,4 +124,44 @@ public abstract class AbstractURLHandler
return new URL(normalizeToString(url));
}
+
+ protected InputStream getDecodingInputStream(String encoding, InputStream
in)
+ throws IOException {
+ InputStream result = null;
+
+ if ("gzip".equals(encoding)) {
+ result = new GZIPInputStream(in);
+ } else if ("deflate".equals(encoding)) {
+ // There seems to be 2 variants of the "deflate"-encoding.
+ // I couldn't find a way to auto-detect which variant was
+ // used, so as (a not really good) work-around we try do
+ // decompress the first 100 bytes using the "zlib"-variant.
+ BufferedInputStream bStream = new BufferedInputStream(in);
+ bStream.mark(100);
+ byte[] bytes = new byte[100];
+ int nbBytes = bStream.read(bytes);
+ bStream.reset();
+
+ Inflater inflater = new Inflater();
+ inflater.setInput(bytes, 0, nbBytes);
+ try {
+ inflater.inflate(new byte[1000]);
+
+ // no error decompressing the first 100 bytes, so we
+ // assume the "zlib"-variant was used.
+ result = new InflaterInputStream(bStream);
+ } catch (DataFormatException e) {
+ // there was an error decompressing the first 100 bytes,
+ // so we assume the "gzip/raw"-variant was used.
+ result = new InflaterInputStream(bStream, new Inflater(true));
+ } finally {
+ inflater.end();
+ }
+ } else {
+ result = in;
+ }
+
+ return result;
+ }
+
}
Modified:
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
(original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
Sat Jun 5 21:25:26 2010
@@ -28,7 +28,6 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.UnknownHostException;
-import java.util.zip.GZIPInputStream;
import org.apache.ivy.Ivy;
import org.apache.ivy.util.CopyProgressListener;
@@ -117,7 +116,7 @@ public class BasicURLHandler extends Abs
url = normalizeToURL(url);
conn = url.openConnection();
conn.setRequestProperty("User-Agent", "Apache Ivy/" +
Ivy.getIvyVersion());
- conn.setRequestProperty("Accept-Encoding", "gzip");
+ conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
if (conn instanceof HttpURLConnection) {
HttpURLConnection httpCon = (HttpURLConnection) conn;
if (!checkStatusCode(url, httpCon)) {
@@ -126,12 +125,8 @@ public class BasicURLHandler extends Abs
+ " See log for more detail.");
}
}
- InputStream inStream;
- if ("gzip".equals(conn.getContentEncoding())) {
- inStream = new GZIPInputStream(conn.getInputStream());
- } else {
- inStream = conn.getInputStream();
- }
+ InputStream inStream =
getDecodingInputStream(conn.getContentEncoding(),
+
conn.getInputStream());
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
byte[] buffer = new byte[BUFFER_SIZE];
@@ -154,7 +149,7 @@ public class BasicURLHandler extends Abs
src = normalizeToURL(src);
srcConn = src.openConnection();
srcConn.setRequestProperty("User-Agent", "Apache Ivy/" +
Ivy.getIvyVersion());
- srcConn.setRequestProperty("Accept-Encoding", "gzip");
+ srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate");
if (srcConn instanceof HttpURLConnection) {
HttpURLConnection httpCon = (HttpURLConnection) srcConn;
if (!checkStatusCode(src, httpCon)) {
@@ -163,23 +158,24 @@ public class BasicURLHandler extends Abs
+ " See log for more detail.");
}
}
- int contentLength = srcConn.getContentLength();
-
- InputStream inStream;
- if ("gzip".equals(srcConn.getContentEncoding())) {
- inStream = new GZIPInputStream(srcConn.getInputStream());
- contentLength = -1;
- } else {
- inStream = srcConn.getInputStream();
- }
+ // do the download
+ InputStream inStream =
getDecodingInputStream(srcConn.getContentEncoding(),
+
srcConn.getInputStream());
FileUtil.copy(inStream, dest, l);
- if (dest.length() != contentLength && contentLength != -1) {
- dest.delete();
- throw new IOException(
- "Downloaded file size doesn't match expected Content
Length for " + src
- + ". Please retry.");
+
+ // check content length only if content was not encoded
+ if (srcConn.getContentEncoding() == null) {
+ int contentLength = srcConn.getContentLength();
+ if (contentLength != -1 && dest.length() != contentLength) {
+ dest.delete();
+ throw new IOException(
+ "Downloaded file size doesn't match expected
Content Length for " + src
+ + ". Please retry.");
+ }
}
+
+ // update modification date
long lastModified = srcConn.getLastModified();
if (lastModified > 0) {
dest.setLastModified(lastModified);
Modified:
ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
(original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
Sat Jun 5 21:25:26 2010
@@ -28,7 +28,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
-import java.util.zip.GZIPInputStream;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
@@ -101,7 +100,10 @@ public class HttpClientHandler extends A
throw new IOException("The HTTP response code for " + url
+ " did not indicate a success." + " See log for more
detail.");
}
- return new GETInputStream(get);
+
+ Header encoding = get.getResponseHeader("Content-Encoding");
+ return getDecodingInputStream(encoding == null ? null :
encoding.getValue(),
+ get.getResponseBodyAsStream());
}
public void download(URL src, File dest, CopyProgressListener l) throws
IOException {
@@ -113,14 +115,9 @@ public class HttpClientHandler extends A
+ " did not indicate a success." + " See log for more
detail.");
}
- InputStream is;
- Header[] contentEncodings =
get.getResponseHeaders("Content-Encoding");
- if ((contentEncodings.length > 0) &&
"gzip".equals(contentEncodings[0].getValue())) {
- is = new GZIPInputStream(get.getResponseBodyAsStream());
- } else {
- is = get.getResponseBodyAsStream();
- }
-
+ Header encoding = get.getResponseHeader("Content-Encoding");
+ InputStream is = getDecodingInputStream(encoding == null ? null :
encoding.getValue(),
+
get.getResponseBodyAsStream());
FileUtil.copy(is, dest, l);
dest.setLastModified(getLastModified(get));
} finally {
@@ -255,7 +252,7 @@ public class HttpClientHandler extends A
GetMethod get = new GetMethod(normalizeToString(url));
get.setDoAuthentication(useAuthentication(url) ||
useProxyAuthentication());
- get.setRequestHeader("Accept-Encoding", "gzip");
+ get.setRequestHeader("Accept-Encoding", "gzip,deflate");
client.executeMethod(get);
return get;
}
@@ -321,72 +318,6 @@ public class HttpClientHandler extends A
return (proxyUserName != null && proxyUserName.trim().length() > 0);
}
- private static final class GETInputStream extends InputStream {
- private InputStream is;
-
- private GetMethod get;
-
- private GETInputStream(GetMethod get) throws IOException {
- this.get = get;
-
- Header[] contentEncodings =
get.getResponseHeaders("Content-Encoding");
- if ((contentEncodings.length > 0) &&
"gzip".equals(contentEncodings[0].getValue())) {
- is = new GZIPInputStream(get.getResponseBodyAsStream());
- } else {
- is = get.getResponseBodyAsStream();
- }
- }
-
- public int available() throws IOException {
- return is.available();
- }
-
- public void close() throws IOException {
- is.close();
- get.releaseConnection();
- }
-
- public boolean equals(Object obj) {
- return is.equals(obj);
- }
-
- public int hashCode() {
- return is.hashCode();
- }
-
- public void mark(int readlimit) {
- is.mark(readlimit);
- }
-
- public boolean markSupported() {
- return is.markSupported();
- }
-
- public int read() throws IOException {
- return is.read();
- }
-
- public int read(byte[] b, int off, int len) throws IOException {
- return is.read(b, off, len);
- }
-
- public int read(byte[] b) throws IOException {
- return is.read(b);
- }
-
- public void reset() throws IOException {
- is.reset();
- }
-
- public long skip(long n) throws IOException {
- return is.skip(n);
- }
-
- public String toString() {
- return is.toString();
- }
- }
-
private static final class HttpClientHelper3x implements HttpClientHelper {
private static final int VERSION = 3;
Modified:
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
---
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
(original)
+++
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
Sat Jun 5 21:25:26 2010
@@ -20,6 +20,8 @@ package org.apache.ivy.util.url;
import java.io.File;
import java.net.URL;
+import org.apache.ivy.util.FileUtil;
+
import junit.framework.TestCase;
/**
@@ -27,8 +29,21 @@ import junit.framework.TestCase;
*/
public class BasicURLHandlerTest extends TestCase {
// remote.test
+ private File testDir;
+ private BasicURLHandler handler;
+
+ protected void setUp() throws Exception {
+ testDir = new File("build/BasicURLHandlerTest");
+ testDir.mkdirs();
+
+ handler = new BasicURLHandler();
+ }
+
+ protected void tearDown() throws Exception {
+ FileUtil.forceDelete(testDir);
+ }
+
public void testIsReachable() throws Exception {
- URLHandler handler = new BasicURLHandler();
assertTrue(handler.isReachable(new URL("http://www.google.fr/";)));
assertFalse(handler.isReachable(new
URL("http://www.google.fr/unknownpage.html";)));
@@ -39,4 +54,19 @@ public class BasicURLHandlerTest extends
// assertTrue(handler.isReachable(new
URL("ftp://ftp.mozilla.org/pub/dir.sizes";)));
assertFalse(handler.isReachable(new
URL("ftp://ftp.mozilla.org/unknown.file";)));
}
+
+ public void testContentEncoding() throws Exception {
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html";), new File(testDir,
"gzip.txt"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on";), new
File(testDir, "deflate-zlib.txt"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on";), new
File(testDir, "deflate.txt"));
+ assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps";),
new File(testDir, "a5-gzip.ps"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on";), new File(testDir,
"a5-deflate.ps"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf";), new File(testDir,
"nh80-gzip.pdf"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on";), new
File(testDir, "nh80-deflate.pdf"));
+ }
+
+ private void assertDownloadOK(URL url, File file) throws Exception {
+ handler.download(url, file, null);
+ assertTrue(file.exists());
+ }
}
Modified:
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java
URL:
http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
---
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java
(original)
+++
ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java
Sat Jun 5 21:25:26 2010
@@ -17,8 +17,10 @@
*/
package org.apache.ivy.util.url;
+import java.io.File;
import java.net.URL;
+import org.apache.ivy.util.FileUtil;
import org.apache.ivy.util.url.URLHandler.URLInfo;
import junit.framework.TestCase;
@@ -28,8 +30,21 @@ import junit.framework.TestCase;
*/
public class HttpclientURLHandlerTest extends TestCase {
// remote.test
+ private File testDir;
+ private HttpClientHandler handler;
+
+ protected void setUp() throws Exception {
+ testDir = new File("build/HttpclientURLHandlerTest");
+ testDir.mkdirs();
+
+ handler = new HttpClientHandler();
+ }
+
+ protected void tearDown() throws Exception {
+ FileUtil.forceDelete(testDir);
+ }
+
public void testIsReachable() throws Exception {
- URLHandler handler = new HttpClientHandler();
assertTrue(handler.isReachable(new URL("http://www.google.fr/";)));
assertFalse(handler.isReachable(new
URL("http://www.google.fr/unknownpage.html";)));
}
@@ -43,4 +58,19 @@ public class HttpclientURLHandlerTest ex
assertEquals(URLHandler.UNAVAILABLE, info);
}
+
+ public void testContentEncoding() throws Exception {
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html";), new File(testDir,
"gzip.txt"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on";), new
File(testDir, "deflate-zlib.txt"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on";), new
File(testDir, "deflate.txt"));
+ assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps";),
new File(testDir, "a5-gzip.ps"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on";), new File(testDir,
"a5-deflate.ps"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf";), new File(testDir,
"nh80-gzip.pdf"));
+ assertDownloadOK(new
URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on";), new
File(testDir, "nh80-deflate.pdf"));
+ }
+
+ private void assertDownloadOK(URL url, File file) throws Exception {
+ handler.download(url, file, null);
+ assertTrue(file.exists());
+ }
}
