I think you forgot to check in your updated testcases... [EMAIL PROTECTED] wrote: > Author: hqm > Date: 2007-11-05 12:00:28 -0800 (Mon, 05 Nov 2007) > New Revision: 7132 > > Added: > openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java > Removed: > openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar > Modified: > openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java > > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java > Log: > Change 20071105-hqm-0 by [EMAIL PROTECTED] on 2007-11-05 14:54:40 EST > in /cygdrive/c/users/hqm/openlaszlo/trunk > for http://svn.openlaszlo.org/openlaszlo/trunk > > Summary: put XML parser back into server data proxy pipeline, to do > charset transcoding > > New Features: > > Bugs Fixed: LPP-4924 > > Technical Reviewer: max > QA Reviewer: pablo > Doc Reviewer: > > Details: > > The server data proxy now uses the XMLPULL parser to parse > the data from the backend, in order to use Java to force a translation > into UTF-8 > coding if needed. > > This change also uses a worker Thread to read from the backend, while > simultaneously > pipelining the data back throug the XML PULL parser to the client. > This should improve > response time and also removes a potential memory overflow and DOS > attack on the server. > > Tests: > > test/lfc/data/alldata.lzx > amazon > calendar > > > > > Deleted: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.3.4d_b4.jar > > Added: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar > > > Property changes on: openlaszlo/trunk/WEB-INF/lib/xpp3-1.1.4c.jar > ___________________________________________________________________ > Name: svn:mime-type > + application/octet-stream > > Modified: openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as > =================================================================== > --- openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as > 2007-11-05 19:52:33 UTC (rev 7131) > +++ openlaszlo/trunk/WEB-INF/lps/lfc/kernel/swf/LzLoadQueue.as > 2007-11-05 20:00:28 UTC (rev 7132) > @@ -94,8 +94,10 @@ > */ > LzLoadQueue.XMLOnDataHandler = function (src) { > if (src == undefined) { > - Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL %w, no > data received.", this.url); > - Debug.warn("Failure to load data in serverless apps may be caused by > Flash player security policies. Check your data server crossdomain.xml file"); > + if (!this.proxied) { > + Debug.warn("LzLoadQueue.XMLOnDataHandler load failed from URL > %w, no data received.", this.url); > + Debug.warn("Failure to load data in serverless apps may be > caused by Flash player security policies. Check your data server > crossdomain.xml file"); > + } > this.onload(false); > //Debug.write("this.loader.onerror.ready =", > this.loader.onerror.ready); > if (this.loader.onerror.ready) { > > Modified: > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java > =================================================================== > --- > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java > 2007-11-05 19:52:33 UTC (rev 7131) > +++ > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/DataSource.java > 2007-11-05 20:00:28 UTC (rev 7132) > @@ -3,7 +3,7 @@ > * > ****************************************************************************/ > > /* J_LZ_COPYRIGHT_BEGIN > ******************************************************* > -* Copyright 2001-2004 Laszlo Systems, Inc. All Rights Reserved. > * > +* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. > * > * Use is subject to license terms. > * > * J_LZ_COPYRIGHT_END > *********************************************************/ > > @@ -169,7 +169,7 @@ > org.openlaszlo.i18n.LaszloMessages.getMessage( > DataSource.class.getName(),"051018-169", new > Object[] {new Long(size)}) > ); > - res.setContentLength((int)size); > + //res.setContentLength((int)size); > } > > if (doClientCache) { > > Modified: > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java > =================================================================== > --- > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java > 2007-11-05 19:52:33 UTC (rev 7131) > +++ > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HTTPDataSource.java > 2007-11-05 20:00:28 UTC (rev 7132) > @@ -614,215 +614,8 @@ > HTTPDataSource.class.getName(),"051018-592", > new Object[] {new Integer(code), HttpStatus.getStatusText(code)}); > } > > - /** > - * A class for holding on to results of an Http fetch. > - * > - * @author <a href="mailto:[EMAIL PROTECTED]">Eric Bloch</a> > - */ > > - public static class HttpData extends Data { > > - /** response code */ > - public final int code; > - > - /** Http request */ > - public final HttpMethodBase request; > - > - private PatternMatcher pMatcher = new Perl5Matcher(); > - private static final Pattern charsetPattern; > - private static final Pattern declEncodingPattern; > - static { > - try { > - Perl5Compiler compiler = new Perl5Compiler(); > - charsetPattern = compiler.compile(";charset=([^ ]*)"); > - declEncodingPattern = > - compiler.compile("[ \t\r\n]*<[?]xml .*encoding=[\"']([^ > \"']*)[\"'] .*[?]>"); > - } catch (MalformedPatternException e) { > - throw new RuntimeException(e.getMessage()); > - } > - } > - > - /** > - * @param r filled request > - * @param c response code > - */ > - public HttpData(HttpMethodBase r, int c) { > - code = c; > - request = r; > - } > - > - /** > - * @return true if the data was "not modified" > - */ > - public boolean notModified() { > - return code == HttpServletResponse.SC_NOT_MODIFIED; > - } > - > - /** > - * @return the lastModified time of the data > - */ > - public long lastModified() { > - > - Header lastModifiedHdr = request.getResponseHeader( > - LZHttpUtils.LAST_MODIFIED); > - > - if (lastModifiedHdr != null) { > - String lm = lastModifiedHdr.getValue(); > - mLogger.debug( > -/* (non-Javadoc) > - * @i18n.test > - * @org-mes="data with last modified at " + p[0] > - */ > - org.openlaszlo.i18n.LaszloMessages.getMessage( > - HTTPDataSource.class.getName(),"051018-655", > new Object[] {lm}) > -); > - long l = LZHttpUtils.getDate(lm); > - // Truncate to nearest second > - return ((l)/1000L) * 1000L; > - } else { > - mLogger.debug( > -/* (non-Javadoc) > - * @i18n.test > - * @org-mes="data has no mod time" > - */ > - org.openlaszlo.i18n.LaszloMessages.getMessage( > - HTTPDataSource.class.getName(),"051018-667") > -); > - return -1; > - } > - } > - > - /** > - * append response headers > - */ > - public void appendResponseHeadersAsXML(StringBuffer xmlResponse) { > - > - Header[] hedz = request.getResponseHeaders(); > - for (int i = 0; i < hedz.length; i++) { > - String name = hedz[i].getName(); > - if (LZHttpUtils.allowForward(name, null)) { > - xmlResponse.append("<header name=\""+ > XMLUtils.escapeXml( name ) + "\" " > - + "value=\"" + > XMLUtils.escapeXml( hedz[i].getValue() ) + "\" />"); > - } > - } > - } > - > - /** > - * release any resources associated with this data > - */ > - public void release() { > - request.releaseConnection(); > - } > - > - /** > - * @return mime type > - */ > - public String getMimeType() { > - Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_TYPE); > - String contentType = ""; > - if (hdr != null) { > - contentType = hdr.getValue(); > - } > - mLogger.debug( > -/* (non-Javadoc) > - * @i18n.test > - * @org-mes="content type: " + p[0] > - */ > - org.openlaszlo.i18n.LaszloMessages.getMessage( > - HTTPDataSource.class.getName(),"051018-710", > new Object[] {contentType}) > -); > - return contentType; > - } > - > - /** > - * @return string > - */ > - public String getAsString() throws IOException { > - byte rawbytes[] = request.getResponseBody(); > - if (rawbytes == null || rawbytes.length == 0) { > - throw new InterruptedIOException("null http response body"); > - } > - String encoding = "UTF-8"; > - String content = getMimeType(); > - // search for ;charset=XXXX in Content-Type header > - if (pMatcher.matches(content, charsetPattern)) { > - encoding = pMatcher.getMatch().group(1); > - } > - // search for 'encoding' attribute in xml declaration, e.g., > - // <?xml version="1.0" encoding="ISO-8859-1" standalone="no"?> > - > - String decl = getXMLDeclaration(rawbytes); > - if (pMatcher.matches(decl, declEncodingPattern)) { > - encoding = pMatcher.getMatch().group(1); > - //mLogger.debug("parsed data encoding: " + encoding); > - } > - > - return new String(rawbytes, encoding); > - } > - > - /** Returns the first non-whitespace line. > - * > - */ > - String getXMLDeclaration(byte buf[]) { > - String str = new String(buf); > - BufferedReader br = new BufferedReader(new StringReader(str)); > - String line; > - while (true) { > - try { line = br.readLine(); } catch (IOException e) { return > ""; } > - if (line == null) { > - return ""; > - } > - if (line.length() == 0) continue; > - if (line.startsWith("<?xml ")) { > - return line; > - } else { > - return ""; > - } > - } > - } > - > - /** > - * @return input stream > - */ > - public InputStream getInputStream() throws IOException { > - InputStream str = request.getResponseBodyAsStream(); > - if (str == null) { > - throw new IOException( > -/* (non-Javadoc) > - * @i18n.test > - * @org-mes="http response body is null" > - */ > - org.openlaszlo.i18n.LaszloMessages.getMessage( > - HTTPDataSource.class.getName(),"051018-774") > -); > - } > - return str; > - } > - > - /** > - * @return size, if known > - */ > - public long size() { > - Header hdr = > request.getResponseHeader(LZHttpUtils.CONTENT_LENGTH); > - if (hdr != null) { > - String contentLength = hdr.getValue(); > - if (contentLength != null) { > - mLogger.debug( > -/* (non-Javadoc) > - * @i18n.test > - * @org-mes="content length: " + p[0] > - */ > - org.openlaszlo.i18n.LaszloMessages.getMessage( > - HTTPDataSource.class.getName(),"051018-794", > new Object[] {contentLength}) > -); > - int cl = Integer.parseInt(contentLength); > - return cl; > - } > - } > - return -1; > - } > - } > - > public static int getConnectionPoolTimeout() { > return mConnectionPoolTimeout; > } > > Added: > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java > > > Property changes on: > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/HttpData.java > ___________________________________________________________________ > Name: svn:executable > + * > Name: svn:mime-type > + text/plain > Name: svn:eol-style > + native > > Modified: > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java > =================================================================== > --- > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java > 2007-11-05 19:52:33 UTC (rev 7131) > +++ > openlaszlo/trunk/WEB-INF/lps/server/src/org/openlaszlo/data/XMLGrabber.java > 2007-11-05 20:00:28 UTC (rev 7132) > @@ -8,7 +8,7 @@ > * > ****************************************************************************/ > > /* J_LZ_COPYRIGHT_BEGIN > ******************************************************* > -* Copyright 2001-2006 Laszlo Systems, Inc. All Rights Reserved. > * > +* Copyright 2001-2007 Laszlo Systems, Inc. All Rights Reserved. > * > * Use is subject to license terms. > * > * J_LZ_COPYRIGHT_END > *********************************************************/ > > @@ -23,6 +23,7 @@ > import org.apache.log4j.*; > > import org.openlaszlo.utils.ContentEncoding; > +import org.openlaszlo.utils.LZHttpUtils; > > import org.openlaszlo.media.MimeType; > import org.openlaszlo.server.LPS; > @@ -35,7 +36,14 @@ > import org.jdom.input.SAXBuilder; > import org.jdom.output.XMLOutputter; > > +import org.apache.commons.httpclient.*; > +import org.apache.commons.httpclient.methods.*; > +import org.apache.commons.httpclient.util.*; > > +import org.xmlpull.v1.*; > + > + > + > /** > * XML Converter > * > @@ -43,10 +51,33 @@ > public class XMLGrabber extends Converter { > > private static Logger mLogger = Logger.getLogger(XMLGrabber.class); > + private static XmlPullParserFactory factory = null; > > + private static XmlPullParserFactory getXPPFactory () { > + if (factory == null) { > + // Set up the XML Parser factory > + try { > + String sys = null; > + try { > + sys = > System.getProperty(XmlPullParserFactory.PROPERTY_NAME); > + } catch (SecurityException se) { > + } > + factory = XmlPullParserFactory.newInstance(sys, null); > + factory.setNamespaceAware(false); > + factory.setValidating(false); > + } catch (XmlPullParserException e) { > + throw new RuntimeException(e.getMessage()); > + } > + } > + return factory; > + } > + > /** > * Convert incoming XML to ... XML > * > + * This method is called convertToSWF for historical reasons, and nobody > + * has changed the API call name yet. > + * > * A dataset will look like this: > * <resultset> > * <body> > @@ -61,63 +92,193 @@ > * </headers> > * </resultset> > */ > + > public InputStream convertToSWF(Data data, HttpServletRequest req, > HttpServletResponse res) > throws ConversionException, IOException { > > - String body = data.getAsString(); > + try { > + PipedOutputStream pout = new PipedOutputStream(); > + PipedInputStream in = new PipedInputStream(pout); > > - // Get headers > - String sendheaders = req.getParameter("sendheaders"); > - StringBuffer headerbuf = new StringBuffer(); > - headerbuf.append("<headers>\n"); > - if (sendheaders == null || sendheaders.equals("true") ) { > - data.appendResponseHeadersAsXML(headerbuf); > - } > - headerbuf.append("</headers>"); > - String headers = headerbuf.toString(); > + XmlSerializer serializer; > + XmlPullParser parser; > + parser = getXPPFactory().newPullParser(); > + InputStream dstream = data.getInputStream(); > + parser.setInput( dstream, null ); > + serializer = factory.newSerializer(); > + serializer.setOutput(pout , "UTF-8"); > > - if (mLogger.isDebugEnabled()) { > - mLogger.info("Output:" + body.length()); > - mLogger.info("Output:\n" + body); > - mLogger.info("Output Headers:" + headers.length()); > - mLogger.info("Output Headers:\n" + headers); > + HttpMethodBase request = ((HttpData) data).getRequest(); > + > + final String sendheaders = req.getParameter("sendheaders"); > + > + XMLCopyThread worker = new XMLCopyThread(pout, > parser,serializer, request, sendheaders); > + worker.start(); > + > + return in; > + > + } catch (XmlPullParserException ex) { > + throw new ConversionException("Parsing XML: " + ex.getMessage()); > } > + } > > - // Default to true, for back compatibility (sigh) > - boolean trimWhitespace = true; > - String trimval = req.getParameter("trimwhitespace"); > - if ("false".equals(trimval)) { > - trimWhitespace = false; > + // Worker thread to parse XML (which serves to translate obscure > + // charsets to UTF-8) and wrap it in <resultset>, possibly adding > + // proxied HTTP headers from backedn response. > + // This is written to the PipedOutputStream which we were passed. > + class XMLCopyThread extends Thread implements Runnable { > + OutputStream pout = null; > + XmlPullParser parser; > + XmlSerializer serializer; > + HttpMethodBase request; > + String sendheaders = null; > + > + XMLCopyThread( OutputStream pout, XmlPullParser parser, > XmlSerializer serializer, > + HttpMethodBase request, > + String sendheaders) { > + this.pout = pout; > + this.parser = parser; > + this.serializer = serializer; > + this.request = request; > + this.sendheaders = sendheaders; > } > > - boolean compress = "true".equals(req.getParameter("compress")); > - > - // nsprefix now defaults to true > - boolean nsprefix = true; > - if ("false".equals(req.getParameter("nsprefix"))) { > - nsprefix = false; > + public void run() { > + try { > + writeXMLDataToOutputStream(); > + pout.flush(); > + pout.close(); > + } catch (XmlPullParserException ex) { > + throw new RuntimeException(ex); > + } catch (IOException ex) { > + throw new RuntimeException(ex); > + } > } > > - // Need to parse body into a DOM, and add the headers, then re-emit > as XML > - // of the form > + // Generates an XML document with this structure > // <resultset> > - // <body> ... </body> > + // <body> [PROXIED_XMLDOC] </body> > // <headers> ... </headers> > // </resultset> > + void writeXMLDataToOutputStream() throws XmlPullParserException, > IOException { > + //Run through XML PULL parser, to convert to UTF8, and > + // wrap in <resultset> tag, plus optional headers > > - StringBuffer xdata = new StringBuffer(); > - xdata.append("<resultset>\n"); > - xdata.append("<body>\n"); > - String xbody = body.replaceFirst("<[?]xml.*?[?]>",""); > - xdata.append(xbody); > - xdata.append("</body>\n"); > - xdata.append(headers); > - xdata.append("</resultset>\n"); > + // Start a standalone document; > + //serializer.startDocument("UTF-8", Boolean.TRUE); > > - // generate inputstream from DOM > - ByteArrayInputStream dis = new > ByteArrayInputStream((xdata.toString()).getBytes("UTF-8")); > - return dis; > + serializer.startTag("", "resultset"); > + serializer.startTag("", "body"); > + > + parser.nextToken(); // read first token > + > + while (parser.getEventType () != XmlPullParser.END_DOCUMENT) { > + writeToken ( parser.getEventType () ); > + parser.nextToken (); > + } > + > + serializer.endTag("", "body"); > + > + // <headers> ... </headers> > + serializer.startTag("", "headers"); > + > + // Get headers > + if (sendheaders == null || sendheaders.equals("true") ) { > + Header[] hedz = request.getResponseHeaders(); > + for (int i = 0; i < hedz.length; i++) { > + String name = hedz[i].getName(); > + if (LZHttpUtils.allowForward(name, null)) { > + serializer.startTag("", "header"); > > + > + serializer.attribute (null, "name", name); > + serializer.attribute (null, "value", > hedz[i].getValue()); > + serializer.endTag("", "header"); > + } > + } > + } > + serializer.endTag("", "headers"); > + > + serializer.endTag("", "resultset"); > + serializer.endDocument(); > + } > + > + private void writeStartTag () > + throws XmlPullParserException, IOException { > + if (!parser.getFeature > (XmlPullParser.FEATURE_REPORT_NAMESPACE_ATTRIBUTES)) { > + for (int i = parser.getNamespaceCount (parser.getDepth ()-1); > + i <= parser.getNamespaceCount (parser.getDepth ())-1; > i++) { > + serializer.setPrefix > + (parser.getNamespacePrefix (i), > + parser.getNamespaceUri (i)); > + } > + } > + serializer.startTag(parser.getNamespace (), parser.getName ()); > + > + for (int i = 0; i < parser.getAttributeCount (); i++) { > + serializer.attribute > + (parser.getAttributeNamespace (i), > + parser.getAttributeName (i), > + parser.getAttributeValue (i)); > + } > + } > + > + > + private void writeToken (int eventType) > + throws XmlPullParserException, IOException { > + switch (eventType) { > + > + case XmlPullParser.START_TAG: > + writeStartTag (); > + break; > + > + case XmlPullParser.END_TAG: > + serializer.endTag(parser.getNamespace (), parser.getName ()); > + break; > + > + case XmlPullParser.START_DOCUMENT: > + //use Boolean.TRUE to make it standalone > + //Boolean standalone = (Boolean) > parser.getProperty(PROPERTY_XMLDECL_STANDALONE); > + //serializer.startDocument(parser.getInputEncoding(), > standalone); > + break; > + > + case XmlPullParser.END_DOCUMENT: > + //serializer.endDocument(); > + break; > + > + case XmlPullParser.IGNORABLE_WHITESPACE: > + //comment it to remove ignorable whtespaces from XML infoset > + String s = parser.getText (); > + serializer.ignorableWhitespace (s); > + break; > + > + case XmlPullParser.TEXT: > + serializer.text (parser.getText ()); > + break; > + > + case XmlPullParser.ENTITY_REF: > + serializer.entityRef (parser.getName ()); > + break; > + > + case XmlPullParser.CDSECT: > + serializer.cdsect( parser.getText () ); > + break; > + > + case XmlPullParser.PROCESSING_INSTRUCTION: > + // serializer.processingInstruction( parser.getText ()); > + break; > + > + case XmlPullParser.COMMENT: > + //serializer.comment (parser.getText ()); > + break; > + > + case XmlPullParser.DOCDECL: > + // serializer.docdecl (parser.getText ()); > + break; > + } > + } > + > + > } > > /** > > > _______________________________________________ > Laszlo-checkins mailing list > [email protected] > http://www.openlaszlo.org/mailman/listinfo/laszlo-checkins
-- Regards, Max Carlson OpenLaszlo.org _______________________________________________ Laszlo-checkins mailing list [email protected] http://www.openlaszlo.org/mailman/listinfo/laszlo-checkins
