Author: lryan
Date: Tue May 20 16:02:12 2008
New Revision: 658481
URL: http://svn.apache.org/viewvc?rev=658481&view=rev
Log:
Misc fixes for content rewriting.
- Handle non-rewriting of Javascript reponses returned with bad mime-types like
text/html.
- Include some logging information in concatented Javascript
- Dont return empty re-writes.
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/DefaultContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMerger.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/NoOpContentRewriter.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMergerTest.java
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
Tue May 20 16:02:12 2008
@@ -140,7 +140,8 @@
// TODO - Make this sensitive to custom rewriting rules
if (response != null) {
if (request.getOptions().rewriter != null &&
- response.getRewritten() != null) {
+ response.getRewritten() != null &&
+ response.getRewritten().getResponseAsBytes().length > 0) {
return response.getRewritten();
}
return response;
@@ -162,7 +163,7 @@
if (request.getOptions().rewriter != null) {
// TODO - Make this sensitive to different rewriting rules
response.setRewritten(
- request.getOptions().rewriter.rewrite(request.getUri(), response));
+ request.getOptions().rewriter.rewrite(request, response));
}
cache.addResponse(request, response);
return response;
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpRequest.java
Tue May 20 16:02:12 2008
@@ -358,6 +358,7 @@
public boolean ownerSigned = true;
public boolean viewerSigned = true;
public ContentRewriter rewriter = null;
+ public String rewriteMimeType = null;
public Options() {}
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpResponse.java
Tue May 20 16:02:12 2008
@@ -19,19 +19,14 @@
import java.io.ByteArrayInputStream;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import java.util.concurrent.ConcurrentHashMap;
-import java.nio.CharBuffer;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CodingErrorAction;
-import java.nio.charset.CharsetEncoder;
/**
* Represents the results of an HTTP content retrieval operation.
@@ -146,7 +141,7 @@
}
/**
- * @reutrn the content length
+ * @return the content length
*/
public int getContentLength() {
return responseBytes.length;
@@ -184,6 +179,13 @@
}
/**
+ * @return The response as a byte array
+ */
+ public byte[] getResponseAsBytes() {
+ return responseBytes;
+ }
+
+ /**
* @return All headers for this object.
*/
public Map<String, List<String>> getAllHeaders() {
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriter.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriter.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriter.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/ContentRewriter.java
Tue May 20 16:02:12 2008
@@ -17,6 +17,7 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import java.io.Reader;
@@ -30,11 +31,11 @@
/**
* Rewrite the original content located at source
- * @param source Location of the original content
+ * @param request Originating request
* @param original Original content
* @return A rewritten copy of the original or null if no rewriting occurred
*/
- public HttpResponse rewrite(URI source, HttpResponse original);
+ public HttpResponse rewrite(HttpRequest request, HttpResponse original);
/**
* Rewrite the original content located at source
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/DefaultContentRewriter.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/DefaultContentRewriter.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/DefaultContentRewriter.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/DefaultContentRewriter.java
Tue May 20 16:02:12 2008
@@ -17,6 +17,7 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import java.io.ByteArrayOutputStream;
@@ -39,15 +40,19 @@
public DefaultContentRewriter() {
}
- public HttpResponse rewrite(URI source, HttpResponse original) {
+ public HttpResponse rewrite(HttpRequest request, HttpResponse original) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream(
(original.getContentLength() * 110) / 100);
OutputStreamWriter output = new OutputStreamWriter(baos,
original.getEncoding());
- if (rewrite(source,
+ String mimeType = original.getHeader("Content-Type");
+ if (request.getOptions() != null && request.getOptions().rewriteMimeType
!= null) {
+ mimeType = request.getOptions().rewriteMimeType;
+ }
+ if (rewrite(request.getUri(),
new InputStreamReader(original.getResponse(),
original.getEncoding()),
- original.getHeader("Content-Type"),
+ mimeType,
output)) {
return new HttpResponse(original.getHttpStatusCode(),
baos.toByteArray(),
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMerger.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMerger.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMerger.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMerger.java
Tue May 20 16:02:12 2008
@@ -21,6 +21,8 @@
import com.google.caja.lexer.HtmlTokenType;
import com.google.caja.lexer.Token;
+import org.apache.shindig.gadgets.servlet.ProxyHandler;
+
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -36,8 +38,8 @@
*/
public class JavascriptTagMerger implements HtmlTagTransformer {
- @SuppressWarnings("unchecked")
- // Scripts has to hold both URIs and tokens.
+ private final static int MAX_URL_LENGTH = 1500;
+
private final List scripts = new ArrayList();
private final String concatBase;
@@ -52,7 +54,11 @@
* @param relativeUrlBase to resolve relative urls
*/
public JavascriptTagMerger(String concatBase, URI relativeUrlBase) {
- this.concatBase = concatBase;
+ // Force the mime-type to mimic browser expectation so rewriters
+ // can function properly
+ this.concatBase = concatBase
+ + ProxyHandler.REWRITE_MIME_TYPE_PARAM
+ + "=text/javascript&";
this.relativeUrlBase = relativeUrlBase;
}
@@ -109,22 +115,32 @@
return;
}
builder.append("<script src=\"").append(concatBase);
+ int urlStart = builder.length();
+ int paramIndex = 1;
try {
for (int i = 0; i < concat.size(); i++) {
URI srcUrl = concat.get(i);
if (!srcUrl.isAbsolute()) {
srcUrl = relativeUrlBase.resolve(srcUrl);
}
- builder.append(i + 1).append("=")
+ builder.append(paramIndex).append("=")
.append(URLEncoder.encode(srcUrl.toString(), "UTF-8"));
if (i < concat.size() - 1) {
- builder.append("&");
+ if (builder.length() - urlStart > MAX_URL_LENGTH) {
+ paramIndex = 1;
+ builder.append("\" type=\"text/javascript\"></script>\n");
+ builder.append("<script src=\"").append(concatBase);
+ urlStart = builder.length();
+ } else {
+ builder.append("&");
+ paramIndex++;
+ }
}
}
builder.append("\" type=\"text/javascript\"></script>");
concat.clear();
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
+ } catch (UnsupportedEncodingException uee) {
+ throw new RuntimeException(uee);
}
}
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/NoOpContentRewriter.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/NoOpContentRewriter.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/NoOpContentRewriter.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/rewrite/NoOpContentRewriter.java
Tue May 20 16:02:12 2008
@@ -18,6 +18,7 @@
*/
package org.apache.shindig.gadgets.rewrite;
+import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import java.io.Reader;
@@ -32,7 +33,7 @@
public NoOpContentRewriter() {
}
- public HttpResponse rewrite(URI source, HttpResponse original) {
+ public HttpResponse rewrite(HttpRequest request, HttpResponse original) {
return null;
}
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
Tue May 20 16:02:12 2008
@@ -27,9 +27,11 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpServletResponseWrapper;
/**
* Servlet which concatenates the content of several proxied HTTP responses
@@ -51,25 +53,35 @@
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse
response)
throws IOException {
+ ResponseWrapper wrapper = new ResponseWrapper(response);
for (int i = 1; i < Integer.MAX_VALUE; i++) {
String url = request.getParameter(Integer.toString(i));
if (url == null) {
break;
}
try {
- proxyHandler.fetch(new RequestWrapper(request, url), response);
+ wrapper.getOutputStream().println("/* ---- Start " + url + " ---- */");
+ proxyHandler.fetch(new RequestWrapper(request, url), wrapper);
+ wrapper.getOutputStream().println("/* ---- End " + url + " ---- */");
} catch (GadgetException ge) {
- outputError(ge, response);
+ if (ge.getCode() != GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT) {
+ outputError(ge, url, response);
+ return;
+ } else {
+ wrapper.getOutputStream().println("/* ---- End " + url + " 404 ----
*/");
+ }
}
}
+ response.setStatus(200);
}
-
- private void outputError(GadgetException excep, HttpServletResponse resp)
+ private void outputError(GadgetException excep, String url,
HttpServletResponse resp)
throws IOException {
StringBuilder err = new StringBuilder();
err.append(excep.getCode().toString());
- err.append(' ');
+ err.append(" concat(");
+ err.append(url);
+ err.append(") ");
err.append(excep.getMessage());
// Log the errors here for now. We might want different severity levels
@@ -96,5 +108,29 @@
return super.getParameter(paramName);
}
}
+
+ /**
+ * Wrap the response to prevent writing through of the status code and
+ * to hold a reference to the stream across multiple proxied parts
+ */
+ private class ResponseWrapper extends HttpServletResponseWrapper {
+
+ private ServletOutputStream outputStream;
+
+ private ResponseWrapper(HttpServletResponse httpServletResponse) {
+ super(httpServletResponse);
+ }
+
+ @Override
+ public ServletOutputStream getOutputStream() throws IOException {
+ if (outputStream == null) {
+ outputStream = super.getOutputStream();
+ }
+ return outputStream;
+ }
+
+ public void setStatus(int i) {
+ }
+ }
}
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ProxyHandler.java
Tue May 20 16:02:12 2008
@@ -18,6 +18,8 @@
*/
package org.apache.shindig.gadgets.servlet;
+import com.google.inject.Inject;
+
import org.apache.shindig.common.SecurityToken;
import org.apache.shindig.common.SecurityTokenDecoder;
import org.apache.shindig.common.SecurityTokenException;
@@ -32,9 +34,6 @@
import org.apache.shindig.gadgets.rewrite.ContentRewriter;
import org.apache.shindig.gadgets.spec.Auth;
import org.apache.shindig.gadgets.spec.Preload;
-
-import com.google.inject.Inject;
-
import org.json.JSONException;
import org.json.JSONObject;
@@ -63,6 +62,7 @@
public static final String SECURITY_TOKEN_PARAM = "st";
public static final String HEADERS_PARAM = "headers";
public static final String NOCACHE_PARAM = "nocache";
+ public static final String REWRITE_MIME_TYPE_PARAM = "rewriteMime";
public static final String SIGN_VIEWER = "signViewer";
public static final String SIGN_OWNER = "signOwner";
public static final String URL_PARAM = "url";
@@ -214,6 +214,11 @@
}
options.rewriter = rewriter;
+ // Allow the rewriter to use an externally forced mime type. This is
needed
+ // allows proper rewriting of <script src="x"/> where x is returned with
+ // a content type like text/html which unfortunately happens all too
often
+ options.rewriteMimeType = request.getParameter(REWRITE_MIME_TYPE_PARAM);
+
return new HttpRequest(
method, url, headers, postBody, options);
} catch (UnsupportedEncodingException e) {
Modified:
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMergerTest.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMergerTest.java?rev=658481&r1=658480&r2=658481&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMergerTest.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/rewrite/JavascriptTagMergerTest.java
Tue May 20 16:02:12 2008
@@ -78,7 +78,7 @@
public void testSingleScriptReWrite() {
String original = "<script src=\"http://a.b.com/1.js\";></script>";
String rewritten
- = "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
+ = "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
validateRewritten(original, rewritten);
}
@@ -86,7 +86,7 @@
String original = "<script src=\"http://a.b.com/1.js\";></script>\n"
+ "<script src=\"http://a.b.com/2.js\";></script>";
String rewritten
- = "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>";
+ = "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>";
validateRewritten(original, rewritten);
}
@@ -102,7 +102,7 @@
String rewritten = "<script type=\"text/javascript\">\n"
+ "doSomething\n"
+ "</script>"
- + "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>"
+ + "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>"
+ "<script type=\"text/javascript\">\n"
+ "doSomething\n"
+ "</script>";
@@ -116,23 +116,23 @@
+ "<script src=\"http://a.b.com/3.js\";></script>\n"
+ "<script src=\"http://a.b.com/4.js\";></script>";
String rewritten =
- "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>"
+ "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js&2=http%3A%2F%2Fa.b.com%2F2.js\";
type=\"text/javascript\"></script>"
+ "<script type=\"text/javascript\"><!-- doSomething --></script>"
- + "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F3.js&2=http%3A%2F%2Fa.b.com%2F4.js\";
type=\"text/javascript\"></script>";
+ + "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F3.js&2=http%3A%2F%2Fa.b.com%2F4.js\";
type=\"text/javascript\"></script>";
validateRewritten(original, rewritten);
}
public void testDerelativizeHostRelative() {
String original = "<script src=\"/1.js\"></script>";
String rewritten
- = "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
+ = "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
validateRewritten(original, rewritten);
}
public void testDerelativizePathRelative() {
String original = "<script src=\"1.js\"></script>";
String rewritten
- = "<script
src=\"http://www.test.com/concat?1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
+ = "<script
src=\"http://www.test.com/concat?rewriteMime=text/javascript&1=http%3A%2F%2Fa.b.com%2F1.js\";
type=\"text/javascript\"></script>";
validateRewritten(original, rewritten);
}
