Author: lryan
Date: Wed May 21 12:21:13 2008
New Revision: 658818
URL: http://svn.apache.org/viewvc?rev=658818&view=rev
Log:
Add LRU caching of specs and message bundles
Move content rewriting into caching layer
Modified:
incubator/shindig/trunk/java/gadgets/conf/gadgets.properties
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicGadgetSpecFactory.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicMessageBundleFactory.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/SigningFetcher.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/AbstractHttpCache.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpCache.java
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/GadgetTestFixture.java
Modified: incubator/shindig/trunk/java/gadgets/conf/gadgets.properties
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/conf/gadgets.properties?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/conf/gadgets.properties (original)
+++ incubator/shindig/trunk/java/gadgets/conf/gadgets.properties Wed May 21
12:21:13 2008
@@ -9,3 +9,5 @@
locked-domain.embed-host=127.0.0.1:8080
content-rewrite.enabled=true
cache.capacity=10000
+gadget-spec.cache.capacity=0
+message-bundle.cache.capacity=0
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicGadgetSpecFactory.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicGadgetSpecFactory.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicGadgetSpecFactory.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicGadgetSpecFactory.java
Wed May 21 12:21:13 2008
@@ -18,6 +18,12 @@
*/
package org.apache.shindig.gadgets;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+
+import org.apache.shindig.common.cache.Cache;
+import org.apache.shindig.common.cache.LruCache;
import org.apache.shindig.gadgets.http.HttpFetcher;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
@@ -26,11 +32,9 @@
import org.apache.shindig.gadgets.spec.GadgetSpec;
import org.apache.shindig.gadgets.spec.View;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-
import java.net.URI;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Basic implementation of a gadget spec factory
@@ -38,10 +42,17 @@
@Singleton
public class BasicGadgetSpecFactory implements GadgetSpecFactory {
+ private static final Logger logger
+ = Logger.getLogger(BasicGadgetSpecFactory.class.getName());
+
private final HttpFetcher specFetcher;
+
private final ContentRewriter rewriter;
+
private final boolean enableRewrite;
+ private final Cache<URI, GadgetSpec> inMemorySpecCache;
+
public GadgetSpec getGadgetSpec(GadgetContext context)
throws GadgetException {
return getGadgetSpec(context.getUrl(), context.getIgnoreCache());
@@ -49,22 +60,41 @@
public GadgetSpec getGadgetSpec(URI gadgetUri, boolean ignoreCache)
throws GadgetException {
- HttpRequest request = HttpRequest.getRequest(
- gadgetUri, ignoreCache);
- HttpResponse response = specFetcher.fetch(request);
- if (response.getHttpStatusCode() != HttpResponse.SC_OK) {
- throw new GadgetException(
- GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT,
- "Unable to retrieve gadget xml. HTTP error " +
- response.getHttpStatusCode());
+ GadgetSpec spec;
+ synchronized (inMemorySpecCache) {
+ spec = inMemorySpecCache.getElement(gadgetUri);
}
- GadgetSpec spec
- = new GadgetSpec(gadgetUri, response.getResponseAsString());
- if (new ContentRewriterFeature(spec, enableRewrite).isRewriteEnabled()) {
- for (View v : spec.getViews().values()) {
- if (v.getType() == View.ContentType.HTML && rewriter != null) {
- v.setRewrittenContent(
- rewriter.rewrite(gadgetUri, v.getContent(), "text/html"));
+ if (ignoreCache || spec == null) {
+ try {
+ HttpRequest request = HttpRequest.getRequest(
+ gadgetUri, ignoreCache);
+ HttpResponse response = specFetcher.fetch(request);
+ if (response.getHttpStatusCode() != HttpResponse.SC_OK) {
+ throw new GadgetException(
+ GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT,
+ "Unable to retrieve gadget xml. HTTP error " +
+ response.getHttpStatusCode());
+ } else {
+ spec = new GadgetSpec(gadgetUri, response.getResponseAsString());
+ if (new ContentRewriterFeature(spec,
enableRewrite).isRewriteEnabled()) {
+ for (View v : spec.getViews().values()) {
+ if (v.getType() == View.ContentType.HTML && rewriter != null) {
+ v.setRewrittenContent(
+ rewriter.rewrite(gadgetUri, v.getContent(), "text/html"));
+ }
+ }
+ }
+ // Add the updated spec back to the cache
+ synchronized (inMemorySpecCache) {
+ inMemorySpecCache.addElement(gadgetUri, spec);
+ }
+ }
+ } catch (GadgetException ge) {
+ if (spec == null) {
+ throw ge;
+ } else {
+ logger.log(Level.WARNING,
+ "Gadget spec fetch failed for " + gadgetUri + " - using cached
", ge);
}
}
}
@@ -74,9 +104,11 @@
@Inject
public BasicGadgetSpecFactory(HttpFetcher specFetcher,
ContentRewriter rewriter,
- @Named("content-rewrite.enabled") boolean defaultEnableRewrite) {
+ @Named("content-rewrite.enabled")boolean defaultEnableRewrite,
+ @Named("gadget-spec.cache.capacity")int gadgetSpecCacheCapacity) {
this.specFetcher = specFetcher;
this.rewriter = rewriter;
this.enableRewrite = defaultEnableRewrite;
+ inMemorySpecCache = new LruCache<URI, GadgetSpec>(gadgetSpecCacheCapacity);
}
}
\ No newline at end of file
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicMessageBundleFactory.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicMessageBundleFactory.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicMessageBundleFactory.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/BasicMessageBundleFactory.java
Wed May 21 12:21:13 2008
@@ -18,22 +18,32 @@
*/
package org.apache.shindig.gadgets;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
+import org.apache.shindig.common.cache.Cache;
+import org.apache.shindig.common.cache.LruCache;
import org.apache.shindig.gadgets.http.HttpFetcher;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import org.apache.shindig.gadgets.spec.LocaleSpec;
import org.apache.shindig.gadgets.spec.MessageBundle;
-import com.google.inject.Inject;
-
import java.net.URI;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Basic implementation of a message bundle factory
*/
public class BasicMessageBundleFactory implements MessageBundleFactory {
- private HttpFetcher bundleFetcher;
+ private static final Logger logger
+ = Logger.getLogger(BasicMessageBundleFactory.class.getName());
+
+ private final HttpFetcher bundleFetcher;
+
+ private final Cache<URI, MessageBundle> inMemoryBundleCache;
public MessageBundle getBundle(LocaleSpec localeSpec, GadgetContext context)
throws GadgetException {
@@ -42,22 +52,41 @@
public MessageBundle getBundle(URI bundleUrl, boolean ignoreCache)
throws GadgetException {
- HttpRequest request
- = HttpRequest.getRequest(bundleUrl, ignoreCache);
- HttpResponse response = bundleFetcher.fetch(request);
- if (response.getHttpStatusCode() != HttpResponse.SC_OK) {
- throw new GadgetException(
- GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT,
- "Unable to retrieve message bundle xml. HTTP error " +
- response.getHttpStatusCode());
+ MessageBundle bundle;
+ synchronized (inMemoryBundleCache) {
+ bundle = inMemoryBundleCache.getElement(bundleUrl);
+ }
+ if (ignoreCache || bundle == null) {
+ try {
+ HttpRequest request
+ = HttpRequest.getRequest(bundleUrl, ignoreCache);
+ HttpResponse response = bundleFetcher.fetch(request);
+ if (response.getHttpStatusCode() != HttpResponse.SC_OK) {
+ throw new GadgetException(
+ GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT,
+ "Unable to retrieve message bundle xml. HTTP error " +
+ response.getHttpStatusCode());
+ }
+ bundle = new MessageBundle(bundleUrl, response.getResponseAsString());
+ synchronized (inMemoryBundleCache) {
+ inMemoryBundleCache.addElement(bundleUrl, bundle);
+ }
+ } catch (GadgetException ge) {
+ if (bundle == null) {
+ throw ge;
+ } else {
+ logger.log(Level.WARNING,
+ "Msg bundle fetch failed for " + bundleUrl + " - using cached
", ge);
+ }
+ }
}
- MessageBundle bundle
- = new MessageBundle(bundleUrl, response.getResponseAsString());
return bundle;
}
@Inject
- public BasicMessageBundleFactory(HttpFetcher bundleFetcher) {
+ public BasicMessageBundleFactory(HttpFetcher bundleFetcher,
+ @Named("message-bundle.cache.capacity")int messageBundleCacheCapacity) {
this.bundleFetcher = bundleFetcher;
+ this.inMemoryBundleCache = new LruCache<URI,
MessageBundle>(messageBundleCacheCapacity);
}
}
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/SigningFetcher.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/SigningFetcher.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/SigningFetcher.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/SigningFetcher.java
Wed May 21 12:21:13 2008
@@ -14,14 +14,6 @@
package org.apache.shindig.gadgets;
-import org.apache.shindig.common.SecurityToken;
-import org.apache.shindig.common.crypto.Crypto;
-import org.apache.shindig.common.util.TimeSource;
-import org.apache.shindig.gadgets.http.HttpCache;
-import org.apache.shindig.gadgets.http.HttpFetcher;
-import org.apache.shindig.gadgets.http.HttpResponse;
-import org.apache.shindig.gadgets.http.HttpRequest;
-
import net.oauth.OAuth;
import net.oauth.OAuth.Parameter;
import net.oauth.OAuthAccessor;
@@ -29,6 +21,14 @@
import net.oauth.OAuthMessage;
import net.oauth.signature.RSA_SHA1;
+import org.apache.shindig.common.SecurityToken;
+import org.apache.shindig.common.crypto.Crypto;
+import org.apache.shindig.common.util.TimeSource;
+import org.apache.shindig.gadgets.http.HttpCache;
+import org.apache.shindig.gadgets.http.HttpFetcher;
+import org.apache.shindig.gadgets.http.HttpRequest;
+import org.apache.shindig.gadgets.http.HttpResponse;
+
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -161,9 +161,7 @@
result = nextFetcher.fetch(signedRequest);
// Try and cache the response
- cache.addResponse(cacheableRequest, result);
-
- return result;
+ return cache.addResponse(cacheableRequest, result);
} catch (GadgetException e) {
throw e;
} catch (Exception e) {
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/AbstractHttpCache.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/AbstractHttpCache.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/AbstractHttpCache.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/AbstractHttpCache.java
Wed May 21 12:21:13 2008
@@ -26,7 +26,8 @@
/**
* Base class for content caches. Defines cache expiration rules and
- * and restrictions on allowed content.
+ * and restrictions on allowed content. Also enforces rewriting
+ * on cacheable content.
*
* TODO: Move cache checking code into HttpUtil
*/
@@ -34,7 +35,8 @@
public final HttpResponse getResponse(HttpRequest request) {
if (canCacheRequest(request)) {
- return getResponse(request.getUri());
+ HttpResponse response = getResponse(request.getUri());
+ return checkRewrite(request, response);
}
return null;
}
@@ -46,17 +48,23 @@
protected abstract HttpResponse getResponseImpl(URI uri);
- public void addResponse(HttpRequest request, HttpResponse response) {
+ public HttpResponse addResponse(HttpRequest request, HttpResponse response) {
if (canCacheRequest(request)) {
+ // !!! Note that we only rewrite cacheable content. Move this call above
the if
+ // to rewrite all content that passes through the cache regardless of
cacheability
+ rewrite(request, response);
addResponse(request.getUri(), response);
+ return checkRewrite(request, response);
}
+ return response;
}
- public void addResponse(URI uri, HttpResponse response) {
+ public HttpResponse addResponse(URI uri, HttpResponse response) {
+ if (uri == null || response == null) return response;
response = checkResponse(response);
- if (uri == null || response == null) return;
// Clone the URI to prevent outside references from preventing collection
addResponseImpl(URI.create(uri.toString()), response);
+ return response;
}
protected abstract void addResponseImpl(URI uri, HttpResponse response);
@@ -164,6 +172,49 @@
}
/**
+ * Add rewritten content to the response if its not there and
+ * we can add it. Re-cache if we created rewritten content.
+ * Return the appropriately re-written version if requested
+ */
+ protected HttpResponse checkRewrite(HttpRequest request, HttpResponse
response) {
+ if (response == null) return null;
+
+ // Perform a rewrite and store the content back to the cache if the
+ // content is actually rewritten
+ if (rewrite(request, response)) {
+ addResponseImpl(request.getUri(), response);
+ }
+
+ // Return the rewritten version if requested
+ if (request.getOptions() != null &&
+ !request.getOptions().ignoreCache &&
+ request.getOptions().rewriter != null &&
+ response.getRewritten() != null &&
+ response.getRewritten().getResponseAsBytes().length > 0) {
+ return response.getRewritten();
+ }
+ return response;
+ }
+
+ /**
+ * Rewrite the content
+ * @return true if rewritten content was generated
+ */
+ protected boolean rewrite(HttpRequest request, HttpResponse response) {
+ if (response == null) return false;
+ // TODO - Make this sensitive to custom rewriting rules
+ if (response.getRewritten() == null &&
+ request.getOptions() != null &&
+ request.getOptions().rewriter != null) {
+ response.setRewritten(request.getOptions().rewriter.rewrite(request,
response));
+ if (response.getRewritten() != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
* Default TTL for an entry in the cache that does not have any
* cache controlling headers
* @return default TTL for cache entries
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
Wed May 21 12:21:13 2008
@@ -137,13 +137,7 @@
/** [EMAIL PROTECTED] */
public HttpResponse fetch(HttpRequest request) {
HttpResponse response = cache.getResponse(request);
- // TODO - Make this sensitive to custom rewriting rules
if (response != null) {
- if (request.getOptions().rewriter != null &&
- response.getRewritten() != null &&
- response.getRewritten().getResponseAsBytes().length > 0) {
- return response.getRewritten();
- }
return response;
}
try {
@@ -160,13 +154,7 @@
fetcher.getOutputStream());
}
response = makeResponse(fetcher);
- if (request.getOptions().rewriter != null) {
- // TODO - Make this sensitive to different rewriting rules
- response.setRewritten(
- request.getOptions().rewriter.rewrite(request, response));
- }
- cache.addResponse(request, response);
- return response;
+ return cache.addResponse(request, response);
} catch (IOException e) {
if (e instanceof FileNotFoundException) {
return HttpResponse.NOT_FOUND;
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpCache.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpCache.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpCache.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/HttpCache.java
Wed May 21 12:21:13 2008
@@ -28,9 +28,9 @@
public HttpResponse getResponse(HttpRequest request);
- public void addResponse(URI uri, HttpResponse response);
+ public HttpResponse addResponse(URI uri, HttpResponse response);
- public void addResponse(HttpRequest request, HttpResponse response);
+ public HttpResponse addResponse(HttpRequest request, HttpResponse response);
public HttpResponse removeResponse(URI uri);
Modified:
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/GadgetTestFixture.java
URL:
http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/GadgetTestFixture.java?rev=658818&r1=658817&r2=658818&view=diff
==============================================================================
---
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/GadgetTestFixture.java
(original)
+++
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/GadgetTestFixture.java
Wed May 21 12:21:13 2008
@@ -40,9 +40,9 @@
public final HttpFetcher fetcher = mock(HttpFetcher.class);
public final GadgetBlacklist blacklist = mock(GadgetBlacklist.class);
public final GadgetSpecFactory specFactory =
- new BasicGadgetSpecFactory(fetcher, new NoOpContentRewriter(), true);
+ new BasicGadgetSpecFactory(fetcher, new NoOpContentRewriter(), true, 0);
public final MessageBundleFactory bundleFactory =
- new BasicMessageBundleFactory(fetcher);
+ new BasicMessageBundleFactory(fetcher, 0);
public GadgetFeatureRegistry registry;
public ContainerConfig containerConfig;
public final Executor executor = new Executor() {
