Hi,
2016-03-08 22:45 GMT+02:00 <ma...@apache.org>:
>
> Author: markt
> Date: Tue Mar 8 20:45:57 2016
> New Revision: 1734150
>
> URL: http://svn.apache.org/viewvc?rev=1734150&view=rev
> Log:
> Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=59017
> Make the pre-compressed file support in the Default Servlet generic so
any compression may be used rather than just gzip.
> Patch provided by Mikko Tiihonen.
> This closes #28
>
> Modified:
> tomcat/trunk/java/org/apache/catalina/servlets/DefaultServlet.java
> tomcat/trunk/test/org/apache/catalina/servlets/TestDefaultServlet.java
> tomcat/trunk/webapps/docs/changelog.xml
> tomcat/trunk/webapps/docs/default-servlet.xml
>
> Modified:
tomcat/trunk/java/org/apache/catalina/servlets/DefaultServlet.java
> URL:
http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/catalina/servlets/DefaultServlet.java?rev=1734150&r1=1734149&r2=1734150&view=diff
>
==============================================================================
> --- tomcat/trunk/java/org/apache/catalina/servlets/DefaultServlet.java
(original)
> +++ tomcat/trunk/java/org/apache/catalina/servlets/DefaultServlet.java
Tue Mar 8 20:45:57 2016
> @@ -36,6 +36,7 @@ import java.util.ArrayList;
> import java.util.Collection;
> import java.util.Enumeration;
> import java.util.Iterator;
> +import java.util.List;
> import java.util.Locale;
> import java.util.StringTokenizer;
>
> @@ -193,9 +194,9 @@ public class DefaultServlet extends Http
> protected boolean readOnly = true;
>
> /**
> - * Should be serve gzip versions of files. By default, it's set to
false.
> + * List of compression formats to serve and their preference order.
> */
> - protected boolean gzip = false;
> + protected CompressionFormat[] compressionFormats;
I think that CompressionFormat should implement also Serializable.
What do you think?
Regards,
Violeta
>
> /**
> * The output buffer size to use when serving resources.
> @@ -280,8 +281,8 @@ public class DefaultServlet extends Http
> if (getServletConfig().getInitParameter("readonly") != null)
> readOnly =
Boolean.parseBoolean(getServletConfig().getInitParameter("readonly"));
>
> - if (getServletConfig().getInitParameter("gzip") != null)
> - gzip =
Boolean.parseBoolean(getServletConfig().getInitParameter("gzip"));
> + compressionFormats =
parseCompressionFormats(getServletConfig().getInitParameter("precompressed"),
> + getServletConfig().getInitParameter("gzip"));
>
> if (getServletConfig().getInitParameter("sendfileSize") != null)
> sendfileSize =
> @@ -321,6 +322,27 @@ public class DefaultServlet extends Http
> }
> }
>
> + private CompressionFormat[] parseCompressionFormats(String
precompressed, String gzip) {
> + List<CompressionFormat> ret = new ArrayList<>();
> + if (precompressed != null && precompressed.indexOf('=') > 0) {
> + for (String pair : precompressed.split(",")) {
> + String[] setting = pair.split("=");
> + String encoding = setting[0];
> + String extension = setting[1];
> + ret.add(new CompressionFormat(extension, encoding));
> + }
> + } else if (precompressed != null) {
> + if (Boolean.parseBoolean(precompressed)) {
> + ret.add(new CompressionFormat(".br", "br"));
> + ret.add(new CompressionFormat(".gz", "gzip"));
> + }
> + } else if (Boolean.parseBoolean(gzip)) {
> + // gzip handling is for backwards compatibility with Tomcat
8.x
> + ret.add(new CompressionFormat(".gz", "gzip"));
> + }
> + return ret.toArray(new CompressionFormat[ret.size()]);
> + }
> +
>
> // ------------------------------------------------------ Protected
Methods
>
> @@ -790,7 +812,7 @@ public class DefaultServlet extends Http
> }
>
> // These need to reflect the original resource, not the
potentially
> - // gzip'd version of the resource so get them now if they are
going to
> + // precompressed version of the resource so get them now if they
are going to
> // be needed later
> String eTag = null;
> String lastModifiedHttp = null;
> @@ -800,11 +822,11 @@ public class DefaultServlet extends Http
> }
>
>
> - // Serve a gzipped version of the file if present
> - boolean usingGzippedVersion = false;
> - if (gzip && !included && resource.isFile() &&
!path.endsWith(".gz")) {
> - WebResource gzipResource = resources.getResource(path +
".gz");
> - if (gzipResource.exists() && gzipResource.isFile()) {
> + // Serve a precompressed version of the file if present
> + boolean usingPrecompressedVersion = false;
> + if (compressionFormats.length > 0 && !included &&
resource.isFile() && !pathEndsWithCompressedExtension(path)) {
> + List<PrecompressedResource> precompressedResources =
getAvailablePrecompressedResources(path);
> + if (!precompressedResources.isEmpty()) {
> Collection<String> varyHeaders =
response.getHeaders("Vary");
> boolean addRequired = true;
> for (String varyHeader : varyHeaders) {
> @@ -817,10 +839,11 @@ public class DefaultServlet extends Http
> if (addRequired) {
> response.addHeader("Vary", "accept-encoding");
> }
> - if (checkIfGzip(request)) {
> - response.addHeader("Content-Encoding", "gzip");
> - resource = gzipResource;
> - usingGzippedVersion = true;
> + PrecompressedResource bestResource =
getBestPrecompressedResource(request, precompressedResources);
> + if (bestResource != null) {
> + response.addHeader("Content-Encoding",
bestResource.format.encoding);
> + resource = bestResource.resource;
> + usingPrecompressedVersion = true;
> }
> }
> }
> @@ -878,7 +901,7 @@ public class DefaultServlet extends Http
> } catch (IllegalStateException e) {
> // If it fails, we try to get a Writer instead if we're
> // trying to serve a text file
> - if (!usingGzippedVersion &&
> + if (!usingPrecompressedVersion &&
> ((contentType == null) ||
> (contentType.startsWith("text")) ||
> (contentType.endsWith("xml")) ||
> @@ -1039,6 +1062,81 @@ public class DefaultServlet extends Http
> }
> }
>
> + private boolean pathEndsWithCompressedExtension(String path) {
> + for (CompressionFormat format : compressionFormats) {
> + if (path.endsWith(format.extension)) {
> + return true;
> + }
> + }
> + return false;
> + }
> +
> + private List<PrecompressedResource>
getAvailablePrecompressedResources(String path) {
> + List<PrecompressedResource> ret = new
ArrayList<>(compressionFormats.length);
> + for (CompressionFormat format : compressionFormats) {
> + WebResource precompressedResource =
resources.getResource(path + format.extension);
> + if (precompressedResource.exists() &&
precompressedResource.isFile()) {
> + ret.add(new PrecompressedResource(precompressedResource,
format));
> + }
> + }
> + return ret;
> + }
> +
> + /**
> + * Match the client preferred encoding formts to the available
precompressed resources.
> + *
> + * @param request The servlet request we are processing
> + * @param precompressedResources List of available precompressed
resources.
> + * @return The best matching precompressed resource or null if no
match was found.
> + */
> + private PrecompressedResource
getBestPrecompressedResource(HttpServletRequest request,
List<PrecompressedResource> precompressedResources) {
> + Enumeration<String> headers =
request.getHeaders("Accept-Encoding");
> + PrecompressedResource bestResource = null;
> + double bestResourceQuality = 0;
> + while (headers.hasMoreElements()) {
> + String header = headers.nextElement();
> + for (String preference : header.split(",")) {
> + if (bestResourceQuality >= 1) {
> + return bestResource;
> + }
> + double quality = 1;
> + int qualityIdx = preference.indexOf(';');
> + if (qualityIdx > 0) {
> + int equalsIdx = preference.indexOf('=', qualityIdx +
1);
> + if (equalsIdx == -1) {
> + continue;
> + }
> + quality =
Double.parseDouble(preference.substring(equalsIdx + 1).trim());
> + }
> + if (quality > bestResourceQuality) {
> + String encoding = preference;
> + if (qualityIdx > 0) {
> + encoding = encoding.substring(0, qualityIdx);
> + }
> + encoding = encoding.trim();
> + if ("identity".equals(encoding)) {
> + bestResource = null;
> + bestResourceQuality = quality;
> + continue;
> + }
> + if ("*".equals(encoding)) {
> + bestResource = precompressedResources.get(0);
> + bestResourceQuality = quality;
> + continue;
> + }
> + for (PrecompressedResource resource :
precompressedResources) {
> + if (encoding.equals(resource.format.encoding)) {
> + bestResource = resource;
> + bestResourceQuality = quality;
> + break;
> + }
> + }
> + }
> + }
> + }
> + return bestResource;
> + }
> +
> private void doDirectoryRedirect(HttpServletRequest request,
HttpServletResponse response)
> throws IOException {
> StringBuilder location = new
StringBuilder(request.getRequestURI());
> @@ -1945,25 +2043,6 @@ public class DefaultServlet extends Http
> }
>
> /**
> - * Check if the user agent supports gzip encoding.
> - *
> - * @param request The servlet request we are processing
> - * @return <code>true</code> if the user agent supports gzip
encoding,
> - * and <code>false</code> if the user agent does not support gzip
encoding.
> - */
> - protected boolean checkIfGzip(HttpServletRequest request) {
> - Enumeration<String> headers =
request.getHeaders("Accept-Encoding");
> - while (headers.hasMoreElements()) {
> - String header = headers.nextElement();
> - if (header.indexOf("gzip") != -1) {
> - return true;
> - }
> - }
> - return false;
> - }
> -
> -
> - /**
> * Check if the if-unmodified-since condition is satisfied.
> *
> * @param request The servlet request we are processing
> @@ -2290,6 +2369,25 @@ public class DefaultServlet extends Http
> }
> }
>
> + protected static class CompressionFormat {
> + public final String extension;
> + public final String encoding;
> +
> + public CompressionFormat(String extension, String encoding) {
> + this.extension = extension;
> + this.encoding = encoding;
> + }
> + }
> +
> + private static class PrecompressedResource {
> + public final WebResource resource;
> + public final CompressionFormat format;
> +
> + private PrecompressedResource(WebResource resource,
CompressionFormat format) {
> + this.resource = resource;
> + this.format = format;
> + }
> + }
>
> /**
> * This is secure in the sense that any attempt to use an external
entity
>
> Modified:
tomcat/trunk/test/org/apache/catalina/servlets/TestDefaultServlet.java
> URL:
http://svn.apache.org/viewvc/tomcat/trunk/test/org/apache/catalina/servlets/TestDefaultServlet.java?rev=1734150&r1=1734149&r2=1734150&view=diff
>
==============================================================================
> ---
tomcat/trunk/test/org/apache/catalina/servlets/TestDefaultServlet.java
(original)
> +++
tomcat/trunk/test/org/apache/catalina/servlets/TestDefaultServlet.java Tue
Mar 8 20:45:57 2016
> @@ -33,6 +33,7 @@ import javax.servlet.http.HttpServletRes
>
> import static org.junit.Assert.assertEquals;
> import static org.junit.Assert.assertFalse;
> +import static org.junit.Assert.assertThat;
> import static org.junit.Assert.assertTrue;
> import static org.junit.Assert.fail;
>
> @@ -40,6 +41,9 @@ import org.junit.Assert;
> import org.junit.Test;
>
> import static org.apache.catalina.startup.SimpleHttpClient.CRLF;
> +import static org.hamcrest.CoreMatchers.containsString;
> +import static org.hamcrest.CoreMatchers.hasItem;
> +import static org.hamcrest.CoreMatchers.not;
>
> import org.apache.catalina.Context;
> import org.apache.catalina.Wrapper;
> @@ -118,19 +122,21 @@ public class TestDefaultServlet extends
>
> tomcat.start();
>
> - TestGzipClient gzipClient = new TestGzipClient(getPort());
> + TestCompressedClient gzipClient = new
TestCompressedClient(getPort());
>
> gzipClient.reset();
> gzipClient.setRequest(new String[] {
> "GET /index.html HTTP/1.1" + CRLF +
> "Host: localhost" + CRLF +
> "Connection: Close" + CRLF +
> - "Accept-Encoding: gzip" + CRLF + CRLF });
> + "Accept-Encoding: gzip, br" + CRLF + CRLF });
> gzipClient.connect();
> gzipClient.processRequest();
> assertTrue(gzipClient.isResponse200());
> List<String> responseHeaders = gzipClient.getResponseHeaders();
> + assertTrue(responseHeaders.contains("Content-Encoding: gzip"));
> assertTrue(responseHeaders.contains("Content-Length: " +
gzipSize));
> + assertTrue(responseHeaders.contains("Vary: accept-encoding"));
>
> gzipClient.reset();
> gzipClient.setRequest(new String[] {
> @@ -144,6 +150,172 @@ public class TestDefaultServlet extends
> assertTrue(responseHeaders.contains("Content-Type: text/html"));
> assertFalse(responseHeaders.contains("Content-Encoding: gzip"));
> assertTrue(responseHeaders.contains("Content-Length: " +
indexSize));
> + assertTrue(responseHeaders.contains("Vary: accept-encoding"));
> + }
> +
> + /*
> + * Verify serving of brotli compressed resources from context root.
> + */
> + @Test
> + public void testBrotliCompressedFile() throws Exception {
> +
> + Tomcat tomcat = getTomcatInstance();
> +
> + File appDir = new File("test/webapp");
> +
> + long brSize = new File(appDir, "index.html.br").length();
> + long indexSize = new File(appDir, "index.html").length();
> +
> + // app dir is relative to server home
> + Context ctxt = tomcat.addContext("", appDir.getAbsolutePath());
> + Wrapper defaultServlet = Tomcat.addServlet(ctxt, "default",
> + "org.apache.catalina.servlets.DefaultServlet");
> + defaultServlet.addInitParameter("precompressed", "true");
> +
> + ctxt.addServletMapping("/", "default");
> + ctxt.addMimeMapping("html", "text/html");
> +
> + tomcat.start();
> +
> + TestCompressedClient client = new
TestCompressedClient(getPort());
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF +
> + "Accept-Encoding: br, gzip" + CRLF + CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + List<String> responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders, hasItem("Content-Encoding: br"));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
brSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF+ CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders, hasItem("Content-Type: text/html"));
> + assertThat(responseHeaders,
not(hasItem(containsString("Content-Encoding"))));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
indexSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> + }
> +
> + /*
> + * Verify serving of custom compressed resources from context root.
> + */
> + @Test
> + public void testCustomCompressedFile() throws Exception {
> +
> + Tomcat tomcat = getTomcatInstance();
> +
> + File appDir = new File("test/webapp");
> +
> + long brSize = new File(appDir, "index.html.br").length();
> + long gzSize = new File(appDir, "index.html.gz").length();
> +
> + // app dir is relative to server home
> + Context ctxt = tomcat.addContext("", appDir.getAbsolutePath());
> + Wrapper defaultServlet = Tomcat.addServlet(ctxt, "default",
> + DefaultServlet.class.getName());
> + defaultServlet.addInitParameter("precompressed",
"gzip=.gz,custom=.br");
> +
> + ctxt.addServletMapping("/", "default");
> + ctxt.addMimeMapping("html", "text/html");
> +
> + tomcat.start();
> +
> + TestCompressedClient client = new
TestCompressedClient(getPort());
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF +
> + "Accept-Encoding: br, gzip ; q = 0.5 , custom" +
CRLF + CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + List<String> responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders, hasItem("Content-Encoding: custom"));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
brSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF +
> + "Accept-Encoding: br;q=1,gzip,custom" + CRLF +
CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders, hasItem("Content-Encoding: gzip"));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
gzSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> + }
> +
> + /*
> + * Verify that "*" and "identity" values are handled correctly in
accept-encoding header.
> + */
> + @Test
> + public void testIdentityAndStarAcceptEncodings() throws Exception {
> +
> + Tomcat tomcat = getTomcatInstance();
> +
> + File appDir = new File("test/webapp");
> +
> + long brSize = new File(appDir, "index.html.br").length();
> + long indexSize = new File(appDir, "index.html").length();
> +
> + // app dir is relative to server home
> + Context ctxt = tomcat.addContext("", appDir.getAbsolutePath());
> + Wrapper defaultServlet = Tomcat.addServlet(ctxt, "default",
> + DefaultServlet.class.getName());
> + defaultServlet.addInitParameter("precompressed",
"br=.br,gzip=.gz");
> +
> + ctxt.addServletMapping("/", "default");
> + ctxt.addMimeMapping("html", "text/html");
> +
> + tomcat.start();
> +
> + TestCompressedClient client = new
TestCompressedClient(getPort());
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF +
> + "Accept-Encoding: gzip;q=0.9,*" + CRLF + CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + List<String> responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders, hasItem("Content-Encoding: br"));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
brSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> +
> + client.reset();
> + client.setRequest(new String[] {
> + "GET /index.html HTTP/1.1" + CRLF +
> + "Host: localhost" + CRLF +
> + "Connection: Close" + CRLF +
> + "Accept-Encoding: gzip;q=0.9,br;q=0,identity," +
CRLF + CRLF });
> + client.connect();
> + client.processRequest();
> + assertTrue(client.isResponse200());
> + responseHeaders = client.getResponseHeaders();
> + assertThat(responseHeaders,
not(hasItem(containsString("Content-Encoding"))));
> + assertThat(responseHeaders, hasItem("Content-Length: " +
indexSize));
> + assertThat(responseHeaders, hasItem("Vary: accept-encoding"));
> }
>
> /*
> @@ -387,9 +559,9 @@ public class TestDefaultServlet extends
> }
> }
>
> - private static class TestGzipClient extends SimpleHttpClient {
> + private static class TestCompressedClient extends SimpleHttpClient {
>
> - public TestGzipClient(int port) {
> + public TestCompressedClient(int port) {
> setPort(port);
> }
>
>
> Modified: tomcat/trunk/webapps/docs/changelog.xml
> URL:
http://svn.apache.org/viewvc/tomcat/trunk/webapps/docs/changelog.xml?rev=1734150&r1=1734149&r2=1734150&view=diff
>
==============================================================================
> --- tomcat/trunk/webapps/docs/changelog.xml (original)
> +++ tomcat/trunk/webapps/docs/changelog.xml Tue Mar 8 20:45:57 2016
> @@ -170,6 +170,11 @@
> related memory leaks when the key class but not the value class
has been
> loaded by the web application class loader. (markt)
> </fix>
> + <add>
> + <bug>59017</bug>: Make the pre-compressed file support in the
Default
> + Servlet generic so any compression may be used rather than just
gzip.
> + Patch provided by Mikko Tiihonen. (markt)
> + </add>
> </changelog>
> </subsection>
> <subsection name="Coyote">
>
> Modified: tomcat/trunk/webapps/docs/default-servlet.xml
> URL:
http://svn.apache.org/viewvc/tomcat/trunk/webapps/docs/default-servlet.xml?rev=1734150&r1=1734149&r2=1734150&view=diff
>
==============================================================================
> --- tomcat/trunk/webapps/docs/default-servlet.xml (original)
> +++ tomcat/trunk/webapps/docs/default-servlet.xml Tue Mar 8 20:45:57 2016
> @@ -94,15 +94,22 @@ directory listings are disabled and debu
> expensive. Multiple requests for large directory listings can
consume
> significant proportions of server resources.
> </property>
> - <property name="gzip">
> - If a gzipped version of a file exists (a file with
<code>.gz</code>
> - appended to the file name located alongside the original file),
Tomcat
> - will serve the gzipped file if the user agent supports gzip and
this
> + <property name="precompressed">
> + If a precompressed version of a file exists (a file with
<code>.br</code>
> + or <code>.gz</code> appended to the file name located alongside
the
> + original file), Tomcat will serve the precompressed file if the
user
> + agent supports the matching content encoding (br or gzip) and
this
> option is enabled. [false]
> <br />
> - The file with the <code>.gz</code> extension will be accessible
if
> - requested directly so if the original resource is protected with
a
> - security constraint, the gzipped version must be similarly
protected.
> + The precompressed file with the with <code>.br</code> or
<code>.gz</code>
> + extension will be accessible if requested directly so if the
original
> + resource is protected with a security constraint, the
precompressed
> + versions must be similarly protected.
> + <br />
> + It is also possible to configure the list of precompressed
formats.
> + The syntax is comma separated list of
> + <code>[content-encoding]=[file-extension]</code> pairs. For
example:
> + <code>br=.br,gzip=.gz,bzip2=.bz2</code>.
> </property>
> <property name="readmeFile">
> If a directory listing is presented, a readme file may also
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org
> For additional commands, e-mail: dev-h...@tomcat.apache.org
>
