Author: hqm
Date: 2007-11-05 12:00:28 -0800 (Mon, 05 Nov 2007)
New Revision: 7132
Added:
openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
Removed:
openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar
Modified:
openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
Log:
Change 20071105-hqm-0 by [EMAIL PROTECTED] on 2007-11-05 14:54:40 EST
in /cygdrive/c/users/hqm/openlaszlo/trunk
for http://svn.openlaszlo.org/openlaszlo/trunk
Summary: put XML parser back into server data proxy pipeline, to do
charset transcoding
New Features:
Bugs Fixed: LPP-4924
Technical Reviewer: max
QA Reviewer: pablo
Doc Reviewer:
Details:
The server data proxy now uses the XMLPULL parser to parse
the data from the backend, in order to use Java to force a translation
into UTF-8
coding if needed.
This change also uses a worker Thread to read from the backend, while
simultaneously
pipelining the data back throug the XML PULL parser to the client.
This should improve
response time and also removes a potential memory overflow and DOS
attack on the server.
Tests:
test/lfc/data/alldata.lzx
amazon
calendar
Deleted: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar
Added: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
Property changes on: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Modified: openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as
===================================================================
--- openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as 2007-11-05
19:52:33 UTC (rev 7131)
+++ openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as 2007-11-05
20:00:28 UTC (rev 7132)
@@ -94,8 +94,10 @@
*/
LzLoadQueue.XMLOnDataHandler = function (src) {
if (src == undefined) {
- Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL %w, no
data received.", this.url);
- Debug.warn("Failure to load data in serverless apps may be caused by
Flash player security policies. Check your data server crossdomain.xml file");
+ if (!this.proxied) {
+ Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL %w,
no data received.", this.url);
+ Debug.warn("Failure to load data in serverless apps may be caused
by Flash player security policies. Check your data server crossdomain.xml
file");
+ }
this.onload(false);
//Debug.write("this.loader.onerror.ready =",
this.loader.onerror.ready);
if (this.loader.onerror.ready) {
Modified:
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
===================================================================
--- openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
2007-11-05 19:52:33 UTC (rev 7131)
+++ openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java
2007-11-05 20:00:28 UTC (rev 7132)
@@ -3,7 +3,7 @@
*
****************************************************************************/
/* J_LZ_COPYRIGHT_BEGIN *******************************************************
-* Copyright 2001-2004 Laszlo Systems, Inc. All Rights Reserved. *
+* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. *
* Use is subject to license terms. *
* J_LZ_COPYRIGHT_END *********************************************************/
@@ -169,7 +169,7 @@
org.openlaszlo.i18n.LaszloMessages.getMessage(
DataSource.class.getName(),"051018-169", new
Object[] {new Long(size)})
);
- res.setContentLength((int)size);
+ //res.setContentLength((int)size);
}
if (doClientCache) {
Modified:
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
===================================================================
---
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
2007-11-05 19:52:33 UTC (rev 7131)
+++
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java
2007-11-05 20:00:28 UTC (rev 7132)
@@ -614,215 +614,8 @@
HTTPDataSource.class.getName(),"051018-592",
new Object[] {new Integer(code), HttpStatus.getStatusText(code)});
}
- /**
- * A class for holding on to results of an Http fetch.
- *
- * @author <a href="mailto:[EMAIL PROTECTED]">Eric Bloch</a>
- */
- public static class HttpData extends Data {
- /** response code */
- public final int code;
-
- /** Http request */
- public final HttpMethodBase request;
-
- private PatternMatcher pMatcher = new Perl5Matcher();
- private static final Pattern charsetPattern;
- private static final Pattern declEncodingPattern;
- static {
- try {
- Perl5Compiler compiler = new Perl5Compiler();
- charsetPattern = compiler.compile(";charset=([^ ]*)");
- declEncodingPattern =
- compiler.compile("[ \t\r\n]*<[?]xml .*encoding=[\"']([^
\"']*)[\"'] .*[?]>");
- } catch (MalformedPatternException e) {
- throw new RuntimeException(e.getMessage());
- }
- }
-
- /**
- * @param r filled request
- * @param c response code
- */
- public HttpData(HttpMethodBase r, int c) {
- code = c;
- request = r;
- }
-
- /**
- * @return true if the data was "not modified"
- */
- public boolean notModified() {
- return code == HttpServletResponse.SC_NOT_MODIFIED;
- }
-
- /**
- * @return the lastModified time of the data
- */
- public long lastModified() {
-
- Header lastModifiedHdr = request.getResponseHeader(
- LZHttpUtils.LAST_MODIFIED);
-
- if (lastModifiedHdr != null) {
- String lm = lastModifiedHdr.getValue();
- mLogger.debug(
-/* (non-Javadoc)
- * @i18n.test
- * @org-mes="data with last modified at " + p[0]
- */
- org.openlaszlo.i18n.LaszloMessages.getMessage(
- HTTPDataSource.class.getName(),"051018-655",
new Object[] {lm})
-);
- long l = LZHttpUtils.getDate(lm);
- // Truncate to nearest second
- return ((l)/1000L) * 1000L;
- } else {
- mLogger.debug(
-/* (non-Javadoc)
- * @i18n.test
- * @org-mes="data has no mod time"
- */
- org.openlaszlo.i18n.LaszloMessages.getMessage(
- HTTPDataSource.class.getName(),"051018-667")
-);
- return -1;
- }
- }
-
- /**
- * append response headers
- */
- public void appendResponseHeadersAsXML(StringBuffer xmlResponse) {
-
- Header[] hedz = request.getResponseHeaders();
- for (int i = 0; i < hedz.length; i++) {
- String name = hedz[i].getName();
- if (LZHttpUtils.allowForward(name, null)) {
- xmlResponse.append("<header name=\""+ XMLUtils.escapeXml(
name ) + "\" "
- + "value=\"" + XMLUtils.escapeXml(
hedz[i].getValue() ) + "\" />");
- }
- }
- }
-
- /**
- * release any resources associated with this data
- */
- public void release() {
- request.releaseConnection();
- }
-
- /**
- * @return mime type
- */
- public String getMimeType() {
- Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_TYPE);
- String contentType = "";
- if (hdr != null) {
- contentType = hdr.getValue();
- }
- mLogger.debug(
-/* (non-Javadoc)
- * @i18n.test
- * @org-mes="content type: " + p[0]
- */
- org.openlaszlo.i18n.LaszloMessages.getMessage(
- HTTPDataSource.class.getName(),"051018-710",
new Object[] {contentType})
-);
- return contentType;
- }
-
- /**
- * @return string
- */
- public String getAsString() throws IOException {
- byte rawbytes[] = request.getResponseBody();
- if (rawbytes == null || rawbytes.length == 0) {
- throw new InterruptedIOException("null http response body");
- }
- String encoding = "UTF-8";
- String content = getMimeType();
- // search for ;charset=XXXX in Content-Type header
- if (pMatcher.matches(content, charsetPattern)) {
- encoding = pMatcher.getMatch().group(1);
- }
- // search for 'encoding' attribute in xml declaration, e.g.,
- // <?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
-
- String decl = getXMLDeclaration(rawbytes);
- if (pMatcher.matches(decl, declEncodingPattern)) {
- encoding = pMatcher.getMatch().group(1);
- //mLogger.debug("parsed data encoding: " + encoding);
- }
-
- return new String(rawbytes, encoding);
- }
-
- /** Returns the first non-whitespace line.
- *
- */
- String getXMLDeclaration(byte buf[]) {
- String str = new String(buf);
- BufferedReader br = new BufferedReader(new StringReader(str));
- String line;
- while (true) {
- try { line = br.readLine(); } catch (IOException e) { return
""; }
- if (line == null) {
- return "";
- }
- if (line.length() == 0) continue;
- if (line.startsWith("<?xml ")) {
- return line;
- } else {
- return "";
- }
- }
- }
-
- /**
- * @return input stream
- */
- public InputStream getInputStream() throws IOException {
- InputStream str = request.getResponseBodyAsStream();
- if (str == null) {
- throw new IOException(
-/* (non-Javadoc)
- * @i18n.test
- * @org-mes="http response body is null"
- */
- org.openlaszlo.i18n.LaszloMessages.getMessage(
- HTTPDataSource.class.getName(),"051018-774")
-);
- }
- return str;
- }
-
- /**
- * @return size, if known
- */
- public long size() {
- Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_LENGTH);
- if (hdr != null) {
- String contentLength = hdr.getValue();
- if (contentLength != null) {
- mLogger.debug(
-/* (non-Javadoc)
- * @i18n.test
- * @org-mes="content length: " + p[0]
- */
- org.openlaszlo.i18n.LaszloMessages.getMessage(
- HTTPDataSource.class.getName(),"051018-794",
new Object[] {contentLength})
-);
- int cl = Integer.parseInt(contentLength);
- return cl;
- }
- }
- return -1;
- }
- }
-
public static int getConnectionPoolTimeout() {
return mConnectionPoolTimeout;
}
Added: openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
Property changes on:
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java
___________________________________________________________________
Name: svn:executable
+ *
Name: svn:mime-type
+ text/plain
Name: svn:eol-style
+ native
Modified:
openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
===================================================================
--- openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
2007-11-05 19:52:33 UTC (rev 7131)
+++ openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java
2007-11-05 20:00:28 UTC (rev 7132)
@@ -8,7 +8,7 @@
* ****************************************************************************/
/* J_LZ_COPYRIGHT_BEGIN *******************************************************
-* Copyright 2001-2006 Laszlo Systems, Inc. All Rights Reserved. *
+* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. *
* Use is subject to license terms. *
* J_LZ_COPYRIGHT_END *********************************************************/
@@ -23,6 +23,7 @@
import org.apache.log4j.*;
import org.openlaszlo.utils.ContentEncoding;
+import org.openlaszlo.utils.LZHttpUtils;
import org.openlaszlo.media.MimeType;
import org.openlaszlo.server.LPS;
@@ -35,7 +36,14 @@
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
+import org.apache.commons.httpclient.*;
+import org.apache.commons.httpclient.methods.*;
+import org.apache.commons.httpclient.util.*;
+import org.xmlpull.v1.*;
+
+
+
/**
* XML Converter
*
@@ -43,10 +51,33 @@
public class XMLGrabber extends Converter {
private static Logger mLogger = Logger.getLogger(XMLGrabber.class);
+ private static XmlPullParserFactory factory = null;
+ private static XmlPullParserFactory getXPPFactory () {
+ if (factory == null) {
+ // Set up the XML Parser factory
+ try {
+ String sys = null;
+ try {
+ sys =
System.getProperty(XmlPullParserFactory.PROPERTY_NAME);
+ } catch (SecurityException se) {
+ }
+ factory = XmlPullParserFactory.newInstance(sys, null);
+ factory.setNamespaceAware(false);
+ factory.setValidating(false);
+ } catch (XmlPullParserException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+ return factory;
+ }
+
/**
* Convert incoming XML to ... XML
*
+ * This method is called convertToSWF for historical reasons, and nobody
+ * has changed the API call name yet.
+ *
* A dataset will look like this:
* <resultset>
* <body>
@@ -61,63 +92,193 @@
* </headers>
* </resultset>
*/
+
public InputStream convertToSWF(Data data, HttpServletRequest req,
HttpServletResponse res)
throws ConversionException, IOException {
- String body = data.getAsString();
+ try {
+ PipedOutputStream pout = new PipedOutputStream();
+ PipedInputStream in = new PipedInputStream(pout);
- // Get headers
- String sendheaders = req.getParameter("sendheaders");
- StringBuffer headerbuf = new StringBuffer();
- headerbuf.append("<headers>\n");
- if (sendheaders == null || sendheaders.equals("true") ) {
- data.appendResponseHeadersAsXML(headerbuf);
- }
- headerbuf.append("</headers>");
- String headers = headerbuf.toString();
+ XmlSerializer serializer;
+ XmlPullParser parser;
+ parser = getXPPFactory().newPullParser();
+ InputStream dstream = data.getInputStream();
+ parser.setInput( dstream, null );
+ serializer = factory.newSerializer();
+ serializer.setOutput(pout , "UTF-8");
- if (mLogger.isDebugEnabled()) {
- mLogger.info("Output:" + body.length());
- mLogger.info("Output:\n" + body);
- mLogger.info("Output Headers:" + headers.length());
- mLogger.info("Output Headers:\n" + headers);
+ HttpMethodBase request = ((HttpData) data).getRequest();
+
+ final String sendheaders = req.getParameter("sendheaders");
+
+ XMLCopyThread worker = new XMLCopyThread(pout, parser,serializer,
request, sendheaders);
+ worker.start();
+
+ return in;
+
+ } catch (XmlPullParserException ex) {
+ throw new ConversionException("Parsing XML: " + ex.getMessage());
}
+ }
- // Default to true, for back compatibility (sigh)
- boolean trimWhitespace = true;
- String trimval = req.getParameter("trimwhitespace");
- if ("false".equals(trimval)) {
- trimWhitespace = false;
+ // Worker thread to parse XML (which serves to translate obscure
+ // charsets to UTF-8) and wrap it in <resultset>, possibly adding
+ // proxied HTTP headers from backedn response.
+ // This is written to the PipedOutputStream which we were passed.
+ class XMLCopyThread extends Thread implements Runnable {
+ OutputStream pout = null;
+ XmlPullParser parser;
+ XmlSerializer serializer;
+ HttpMethodBase request;
+ String sendheaders = null;
+
+ XMLCopyThread( OutputStream pout, XmlPullParser parser, XmlSerializer
serializer,
+ HttpMethodBase request,
+ String sendheaders) {
+ this.pout = pout;
+ this.parser = parser;
+ this.serializer = serializer;
+ this.request = request;
+ this.sendheaders = sendheaders;
}
- boolean compress = "true".equals(req.getParameter("compress"));
-
- // nsprefix now defaults to true
- boolean nsprefix = true;
- if ("false".equals(req.getParameter("nsprefix"))) {
- nsprefix = false;
+ public void run() {
+ try {
+ writeXMLDataToOutputStream();
+ pout.flush();
+ pout.close();
+ } catch (XmlPullParserException ex) {
+ throw new RuntimeException(ex);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
}
- // Need to parse body into a DOM, and add the headers, then re-emit as
XML
- // of the form
+ // Generates an XML document with this structure
// <resultset>
- // <body> ... </body>
+ // <body> [PROXIED_XMLDOC] </body>
// <headers> ... </headers>
// </resultset>
+ void writeXMLDataToOutputStream() throws XmlPullParserException,
IOException {
+ //Run through XML PULL parser, to convert to UTF8, and
+ // wrap in <resultset> tag, plus optional headers
- StringBuffer xdata = new StringBuffer();
- xdata.append("<resultset>\n");
- xdata.append("<body>\n");
- String xbody = body.replaceFirst("<[?]xml.*?[?]>","");
- xdata.append(xbody);
- xdata.append("</body>\n");
- xdata.append(headers);
- xdata.append("</resultset>\n");
+ // Start a standalone document;
+ //serializer.startDocument("UTF-8", Boolean.TRUE);
- // generate inputstream from DOM
- ByteArrayInputStream dis = new
ByteArrayInputStream((xdata.toString()).getBytes("UTF-8"));
- return dis;
+ serializer.startTag("", "resultset");
+ serializer.startTag("", "body");
+
+ parser.nextToken(); // read first token
+
+ while (parser.getEventType () != XmlPullParser.END_DOCUMENT) {
+ writeToken ( parser.getEventType () );
+ parser.nextToken ();
+ }
+
+ serializer.endTag("", "body");
+
+ // <headers> ... </headers>
+ serializer.startTag("", "headers");
+
+ // Get headers
+ if (sendheaders == null || sendheaders.equals("true") ) {
+ Header[] hedz = request.getResponseHeaders();
+ for (int i = 0; i < hedz.length; i++) {
+ String name = hedz[i].getName();
+ if (LZHttpUtils.allowForward(name, null)) {
+ serializer.startTag("", "header");
+
+ serializer.attribute (null, "name", name);
+ serializer.attribute (null, "value",
hedz[i].getValue());
+ serializer.endTag("", "header");
+ }
+ }
+ }
+ serializer.endTag("", "headers");
+
+ serializer.endTag("", "resultset");
+ serializer.endDocument();
+ }
+
+ private void writeStartTag ()
+ throws XmlPullParserException, IOException {
+ if (!parser.getFeature
(XmlPullParser.FEATURE_REPORT_NAMESPACE_ATTRIBUTES)) {
+ for (int i = parser.getNamespaceCount (parser.getDepth ()-1);
+ i <= parser.getNamespaceCount (parser.getDepth ())-1;
i++) {
+ serializer.setPrefix
+ (parser.getNamespacePrefix (i),
+ parser.getNamespaceUri (i));
+ }
+ }
+ serializer.startTag(parser.getNamespace (), parser.getName ());
+
+ for (int i = 0; i < parser.getAttributeCount (); i++) {
+ serializer.attribute
+ (parser.getAttributeNamespace (i),
+ parser.getAttributeName (i),
+ parser.getAttributeValue (i));
+ }
+ }
+
+
+ private void writeToken (int eventType)
+ throws XmlPullParserException, IOException {
+ switch (eventType) {
+
+ case XmlPullParser.START_TAG:
+ writeStartTag ();
+ break;
+
+ case XmlPullParser.END_TAG:
+ serializer.endTag(parser.getNamespace (), parser.getName ());
+ break;
+
+ case XmlPullParser.START_DOCUMENT:
+ //use Boolean.TRUE to make it standalone
+ //Boolean standalone = (Boolean)
parser.getProperty(PROPERTY_XMLDECL_STANDALONE);
+ //serializer.startDocument(parser.getInputEncoding(),
standalone);
+ break;
+
+ case XmlPullParser.END_DOCUMENT:
+ //serializer.endDocument();
+ break;
+
+ case XmlPullParser.IGNORABLE_WHITESPACE:
+ //comment it to remove ignorable whtespaces from XML infoset
+ String s = parser.getText ();
+ serializer.ignorableWhitespace (s);
+ break;
+
+ case XmlPullParser.TEXT:
+ serializer.text (parser.getText ());
+ break;
+
+ case XmlPullParser.ENTITY_REF:
+ serializer.entityRef (parser.getName ());
+ break;
+
+ case XmlPullParser.CDSECT:
+ serializer.cdsect( parser.getText () );
+ break;
+
+ case XmlPullParser.PROCESSING_INSTRUCTION:
+ // serializer.processingInstruction( parser.getText ());
+ break;
+
+ case XmlPullParser.COMMENT:
+ //serializer.comment (parser.getText ());
+ break;
+
+ case XmlPullParser.DOCDECL:
+ // serializer.docdecl (parser.getText ());
+ break;
+ }
+ }
+
+
}
/**
_______________________________________________
Laszlo-checkins mailing list
[email protected]
http://www.openlaszlo.org/mailman/listinfo/laszlo-checkins
