Author: johnh Date: Wed Jul 21 09:54:41 2010 New Revision: 966159 URL: http://svn.apache.org/viewvc?rev=966159&view=rev Log: Workaround fix for JDK ZLIB bug reporting EOFException, provided by Vikas Arora.
Vikas' description: Due to JDK bug (Ref: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4040920), the 'EOF Exception' comes while accessing zip content. This issue is very inconsistent and happens for some zip resources and that too once in 4-5 times. This is little hard to reproduce but when it comes, it comes for wrong case/reason, even though the resource itself is fetched fine. Refer the bug link, mentioned above. The bug lies in the JDK 'Inflater.finished()' call, that erroneously returns 'false' even if the input stream (zip) is done reading the response. Whenever this happens, 'java.util.zip.InflaterInputStream.fill' will throw the 'EOFException'. As a fix for this specific corner case, ignore the Exception (EOF with specific cause trail) and let pass all other exceptions. Verified this working OK on the site for which this error was initially reported. ===== Thanks Vikas! Modified: shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java Modified: shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java URL: http://svn.apache.org/viewvc/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java?rev=966159&r1=966158&r2=966159&view=diff ============================================================================== --- shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java (original) +++ shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java Wed Jul 21 09:54:41 2010 @@ -66,10 +66,12 @@ import org.apache.http.params.HttpConnec import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.protocol.HttpContext; +import org.apache.http.util.ByteArrayBuffer; import org.apache.http.util.EntityUtils; import org.apache.shindig.common.uri.Uri; import org.apache.shindig.gadgets.GadgetException; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.net.ProxySelector; @@ -464,11 +466,72 @@ public class BasicHttpFetcher implements return HttpResponse.badrequest("Exceeded maximum number of bytes - " + maxObjSize); } - byte[] responseBytes = (entity == null) ? null : EntityUtils.toByteArray(entity); + byte[] responseBytes = (entity == null) ? null : toByteArraySafe(entity); return builder .setHttpStatusCode(response.getStatusLine().getStatusCode()) .setResponse(responseBytes) .create(); } + + /** + * This method is Safe replica version of org.apache.http.util.EntityUtils.toByteArray. + * The try block embedding 'instream.read' has a corresponding catch block for 'EOFException' + * (that's Ignored) and all other IOExceptions are let pass. + * + * @param entity + * @return byte array containing the entity content. May be empty/null. + * @throws IOException if an error occurs reading the input stream + */ + public byte[] toByteArraySafe(final HttpEntity entity) throws IOException { + if (entity == null) { + return null; + } + + InputStream instream = entity.getContent(); + if (instream == null) { + return new byte[] {}; + } + if (entity.getContentLength() > Integer.MAX_VALUE) { + throw new IllegalArgumentException("HTTP entity too large to be buffered in memory"); + } + + // The raw data stream (inside JDK) is read in a buffer of size '512'. The original code + // org.apache.http.util.EntityUtils.toByteArray reads the unzipped data in a buffer of + // 4096 byte. For any data stream that has a compression ratio lesser than 1/8, this may + // result in the buffer/array overflow. Increasing the buffer size to '16384'. It's highly + // unlikely to get data compression ratios lesser than 1/32 (3%). + final int bufferLength = 16384; + int i = (int)entity.getContentLength(); + if (i < 0) { + i = bufferLength; + } + ByteArrayBuffer buffer = new ByteArrayBuffer(i); + try { + byte[] tmp = new byte[bufferLength]; + int l; + while((l = instream.read(tmp)) != -1) { + buffer.append(tmp, 0, l); + } + } catch (EOFException eofe) { + // Ref: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4040920 + // Due to a bug in JDK ZLIB (InflaterInputStream), unexpected EOF error can occur. + // In such cases, even if the input stream is finished reading, the + // 'Inflater.finished()' call erroneously returns 'false' and + // 'java.util.zip.InflaterInputStream.fill' throws the 'EOFException'. + // So for such case, ignore the Exception in case Exception Cause is + // 'Unexpected end of ZLIB input stream'. + // For all other cases, re-throw the (EOF) Exception. + if ((instream.available() == 0) && + eofe.getMessage().equals("Unexpected end of ZLIB input stream")) { + // Ignore + } else { + throw eofe; + } + } + finally { + instream.close(); + } + return buffer.toByteArray(); + } } Modified: shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java URL: http://svn.apache.org/viewvc/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java?rev=966159&r1=966158&r2=966159&view=diff ============================================================================== --- shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java (original) +++ shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java Wed Jul 21 09:54:41 2010 @@ -17,22 +17,59 @@ */ package org.apache.shindig.gadgets.http; +import java.io.InputStream; +import java.io.EOFException; +import java.io.IOException; + +import org.apache.http.HttpEntity; import org.apache.shindig.common.uri.Uri; import org.apache.shindig.common.uri.UriBuilder; +import org.easymock.EasyMock; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Before; import org.junit.Test; -public class BasicHttpFetcherTest extends AbstractHttpFetcherTest { +public class BasicHttpFetcherTest { + private static final int ECHO_PORT = 9003; + protected static final Uri BASE_URL = Uri.parse("http://localhost:9003/";); + private static EchoServer server; + + protected BasicHttpFetcher fetcher = null; + protected HttpEntity mockEntity; + protected InputStream mockInputStream; + + @BeforeClass + public static void setUpOnce() throws Exception { + server = new EchoServer(); + server.start(ECHO_PORT); + } + + @AfterClass + public static void tearDownOnce() throws Exception { + if (server != null) { + server.stop(); + } + } + @Before public void setUp() throws Exception { - fetcher = new BasicHttpFetcher(null); + fetcher = new BasicHttpFetcher(BASE_URL.getAuthority()); + + mockInputStream = EasyMock.createMock(InputStream.class); + //EasyMock.expect(mockInputStream.available()).andReturn(0).anyTimes(); + EasyMock.expect(mockInputStream.available()).andReturn(0); + mockInputStream.close(); + + mockEntity = EasyMock.createMock(HttpEntity.class); + EasyMock.expect(mockEntity.getContent()).andReturn(mockInputStream); + EasyMock.expect(mockEntity.getContentLength()).andReturn(new Long(16384)).anyTimes(); } @Test public void testWithProxy() throws Exception { - fetcher = new BasicHttpFetcher(BASE_URL.getAuthority()); - String content = "Hello, Gagan!"; Uri uri = new UriBuilder(Uri.parse("http://www.google.com/search";)) .addQueryParameter("body", content) @@ -43,4 +80,66 @@ public class BasicHttpFetcherTest extend assertEquals(201, response.getHttpStatusCode()); assertEquals(content, response.getResponseAsString()); } + + @Test + public void testToByteArraySafeThrowsException1() throws Exception { + String exceptionMessage = "IO Exception and Any Random Cause"; + IOException e = new IOException(exceptionMessage); + EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes(); + + EasyMock.replay(mockEntity, mockInputStream); + + try { + fetcher.toByteArraySafe(mockEntity); + } catch (IOException ioe) { + assertEquals(exceptionMessage, ioe.getMessage()); + } + } + + @Test + public void testToByteArraySafeThrowsException2() throws Exception { + String exceptionMessage = "EOF Exception and Any Random Cause"; + EOFException e = new EOFException(exceptionMessage); + EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes(); + + EasyMock.replay(mockEntity, mockInputStream); + + try { + fetcher.toByteArraySafe(mockEntity); + } catch (EOFException eofe) { + assertEquals(exceptionMessage, eofe.getMessage()); + } + } + + @Test + public void testToByteArraySafeThrowsException3() throws Exception { + // Return non-zero for 'InputStream.available()'. This should violate the other condition. + EasyMock.expect(mockInputStream.available()).andReturn(1); + String exceptionMessage = "Unexpected end of ZLIB input stream"; + EOFException e = new EOFException(exceptionMessage); + EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes(); + + EasyMock.replay(mockEntity, mockInputStream); + + try { + fetcher.toByteArraySafe(mockEntity); + } catch (EOFException eofe) { + assertEquals(exceptionMessage, eofe.getMessage()); + } + } + + @Test + public void testToByteArraySafeHandleException() throws Exception { + String exceptionMessage = "Unexpected end of ZLIB input stream"; + EOFException e = new EOFException(exceptionMessage); + EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes(); + + EasyMock.replay(mockEntity, mockInputStream); + + try { + fetcher.toByteArraySafe(mockEntity); + } catch (EOFException eofe) { + fail("Exception Should have been caught"); + } + } }
