Author: rdonkin
Date: Tue Jun 24 13:03:44 2008
New Revision: 671327
URL: http://svn.apache.org/viewvc?rev=671327&view=rev
Log:
Parser for structured fields. This should copes with folding and comments.
Added:
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/ParseException.java
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/StructuredFieldParser.jj
james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/
james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/StructuredFieldParserTest.java
Added:
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/ParseException.java
URL:
http://svn.apache.org/viewvc/james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/ParseException.java?rev=671327&view=auto
==============================================================================
---
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/ParseException.java
(added)
+++
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/ParseException.java
Tue Jun 24 13:03:44 2008
@@ -0,0 +1,217 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0
*/
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mime4j.field.structured;
+
+import org.apache.james.mime4j.MimeException;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * This class was modified to extend MimeException
+ * and to add a seriaVersionUID.
+ */
+public class ParseException extends MimeException {
+
+ /**
+ * This constructor is used by the method "generateParseException"
+ * in the generated parser. Calling this constructor generates
+ * a new object of this type with the fields "currentToken",
+ * "expectedTokenSequences", and "tokenImage" set. The boolean
+ * flag "specialConstructor" is also set to true to indicate that
+ * this constructor was used to create this object.
+ * This constructor calls its super class with the empty string
+ * to force the "toString" method of parent class "Throwable" to
+ * print the error message in the form:
+ * ParseException: <result of getMessage>
+ */
+ public ParseException(Token currentTokenVal,
+ int[][] expectedTokenSequencesVal,
+ String[] tokenImageVal
+ )
+ {
+ super("Cannot parse field");
+ specialConstructor = true;
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+
+ /**
+ * The following constructors are for use by you for whatever
+ * purpose you can think of. Constructing the exception in this
+ * manner makes the exception behave in the normal way - i.e., as
+ * documented in the class "Throwable". The fields "errorToken",
+ * "expectedTokenSequences", and "tokenImage" do not contain
+ * relevant information. The JavaCC generated code does not use
+ * these constructors.
+ */
+
+ public ParseException() {
+ super("Cannot parse field");
+ specialConstructor = false;
+ }
+
+ public ParseException(Throwable t) {
+ this(t.getMessage());
+ initCause(t);
+ }
+
+ public ParseException(String message) {
+ super(message);
+ specialConstructor = false;
+ }
+
+ /**
+ * This variable determines which constructor was used to create
+ * this object and thereby affects the semantics of the
+ * "getMessage" method (see below).
+ */
+ protected boolean specialConstructor;
+
+ /**
+ * This is the last token that has been consumed successfully. If
+ * this object has been created due to a parse error, the token
+ * followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+
+ /**
+ * Each entry in this array is an array of integers. Each array
+ * of integers represents a sequence of tokens (by their ordinal
+ * values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+
+ /**
+ * This is a reference to the "tokenImage" array of the generated
+ * parser within which the parse error occurred. This array is
+ * defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+
+ /**
+ * This method has the standard behavior when this object has been
+ * created using the standard constructors. Otherwise, it uses
+ * "currentToken" and "expectedTokenSequences" to generate a parse
+ * error message and returns it. If this object has been created
+ * due to a parse error, and you do not catch it (it gets thrown
+ * from the parser), then this method is called during the printing
+ * of the final stack trace, and hence the correct error message
+ * gets displayed.
+ */
+ public String getMessage() {
+ if (!specialConstructor) {
+ return super.getMessage();
+ }
+ StringBuffer expected = new StringBuffer();
+ int maxSize = 0;
+ for (int i = 0; i < expectedTokenSequences.length; i++) {
+ if (maxSize < expectedTokenSequences[i].length) {
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+ expected.append(tokenImage[expectedTokenSequences[i][j]]).append(" ");
+ }
+ if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] !=
0) {
+ expected.append("...");
+ }
+ expected.append(eol).append(" ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for (int i = 0; i < maxSize; i++) {
+ if (i != 0) retval += " ";
+ if (tok.kind == 0) {
+ retval += tokenImage[0];
+ break;
+ }
+ retval += add_escapes(tok.image);
+ tok = tok.next;
+ }
+ retval += "\" at line " + currentToken.next.beginLine + ", column " +
currentToken.next.beginColumn;
+ retval += "." + eol;
+ if (expectedTokenSequences.length == 1) {
+ retval += "Was expecting:" + eol + " ";
+ } else {
+ retval += "Was expecting one of:" + eol + " ";
+ }
+ retval += expected.toString();
+ return retval;
+ }
+
+ /**
+ * The end of line string for this machine.
+ */
+ protected String eol = System.getProperty("line.separator", "\n");
+
+ /**
+ * Used to convert raw characters to their escaped version
+ * when these raw version cannot be used as part of an ASCII
+ * string literal.
+ */
+ protected String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for (int i = 0; i < str.length(); i++) {
+ switch (str.charAt(i))
+ {
+ case 0 :
+ continue;
+ case '\b':
+ retval.append("\\b");
+ continue;
+ case '\t':
+ retval.append("\\t");
+ continue;
+ case '\n':
+ retval.append("\\n");
+ continue;
+ case '\f':
+ retval.append("\\f");
+ continue;
+ case '\r':
+ retval.append("\\r");
+ continue;
+ case '\"':
+ retval.append("\\\"");
+ continue;
+ case '\'':
+ retval.append("\\\'");
+ continue;
+ case '\\':
+ retval.append("\\\\");
+ continue;
+ default:
+ if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4,
s.length()));
+ } else {
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+
+}
Added:
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/StructuredFieldParser.jj
URL:
http://svn.apache.org/viewvc/james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/StructuredFieldParser.jj?rev=671327&view=auto
==============================================================================
---
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/StructuredFieldParser.jj
(added)
+++
james/mime4j/trunk/src/main/javacc/org/apache/james/mime4j/field/structured/StructuredFieldParser.jj
Tue Jun 24 13:03:44 2008
@@ -0,0 +1,211 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+options {
+ static=false;
+ JDK_VERSION = "1.4";
+ OUTPUT_DIRECTORY =
"../../../../../../../../../target/generated-sources/javacc";
+ //DEBUG_PARSER = true;
+ //DEBUG_TOKEN_MANAGER = true;
+}
+
+PARSER_BEGIN(StructuredFieldParser)
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mime4j.field.structured;
+
+/**
+ * Parses generic structure fields.
+ * Unfolds and removes comments.
+ */
+public class StructuredFieldParser {
+
+ /**
+ * Unfolds the input and removes comments.
+ * @return unfolded header value with comments removed
+ */
+ public String parse() throws ParseException {
+ try {
+ return doParse();
+ } catch (TokenMgrError e) {
+ // An issue with the TOKENiser
+ // but it's not polite to throw an Error
+ // when executing on a server
+ throw new ParseException(e);
+ }
+ }
+}
+PARSER_END(StructuredFieldParser)
+
+private String doParse() :
+{
+ Token t;
+ StringBuffer buffer = new StringBuffer(50);
+ boolean whitespace = false;
+ boolean first = true;
+}
+{
+ (
+ t = <CONTENT>
+ {
+ if (first) {
+ first = false;
+ } else if (whitespace) {
+ buffer.append(" ");
+ whitespace = false;
+ }
+ buffer.append(t.image);
+ }
+ |
+ t = <STRING_CONTENT>
+ {
+ buffer.append(t.image);
+ }
+ |
+ t = <QUOTEDSTRING>
+ {
+ if (first) {
+ first = false;
+ } else if (whitespace) {
+ buffer.append(" ");
+ whitespace = false;
+ }
+ buffer.append(t.image);
+ }
+ |
+ t = <FOLD>
+ {
+ buffer.append("\r\n");
+ }
+ |
+ t = <WS>
+ {
+ whitespace = true;
+ }
+ )*
+ {return buffer.toString();}
+}
+
+TOKEN_MGR_DECLS :
+{
+ // Keeps track of how many levels of comment nesting
+ // we've encountered. This is only used when the 2nd
+ // level is reached, for example ((this)), not (this).
+ // This is because the outermost level must be treated
+ // specially anyway, because the outermost ")" has a
+ // different token type than inner ")" instances.
+ int commentNest;
+}
+
+
+SKIP :
+{
+ // starts a comment
+ "(" : INCOMMENT
+}
+
+<INCOMMENT> SKIP :
+{
+ // ends a comment
+ ")" : DEFAULT
+ // if this is ever changed to not be a SKIP, need
+ // to make sure matchedToken.token = token.toString()
+ // is called.
+}
+
+<INCOMMENT> SKIP :
+{
+ "(" { commentNest = 1; } : NESTED_COMMENT
+}
+
+<INCOMMENT> SKIP :
+{
+ <~[ "(", ")" ]>
+}
+
+<NESTED_COMMENT> SKIP:
+{
+ "(" { ++commentNest; System.out.println("+++ COMMENT NEST=" +
commentNest);}
+| ")" { --commentNest; System.out.println("+++ COMMENT NEST=" +
commentNest); if (commentNest == 0) SwitchTo(INCOMMENT); }
+}
+
+<NESTED_COMMENT> SKIP :
+{
+ < <QUOTEDPAIR>> { image.deleteCharAt(image.length() - 2); }
+}
+
+<NESTED_COMMENT> SKIP:
+{
+ <~[ "(", ")" ]>
+}
+
+// QUOTED STRINGS
+
+SKIP :
+{
+ "\"" : INQUOTEDSTRING
+}
+
+<INQUOTEDSTRING> MORE :
+{
+ < <QUOTEDPAIR> > { image.deleteCharAt(image.length() - 2); }
+}
+
+<INQUOTEDSTRING> TOKEN :
+{
+ < STRING_CONTENT : (~["\"", "\\", "\r"])+ >
+ // Preserve line break within quotes but not trailing white space
+| < FOLD: "\r\n" ( [" ", "\t"] )* >
+| < QUOTEDSTRING: "\"" > { matchedToken.image = image.substring(0,
image.length() - 1); } : DEFAULT
+}
+
+<DEFAULT>
+TOKEN :
+{
+ < WS: ( [" ", "\t", "\r", "\n"] )+ >
+}
+
+<DEFAULT>
+TOKEN :
+{
+ < CONTENT: (~[" ", "\t", "\r", "\n", "(", "\""])+ >
+}
+
+<*>
+TOKEN :
+{
+ < #QUOTEDPAIR: "\\" <ANY> >
+| < #ANY: (~[])+ >
+}
\ No newline at end of file
Added:
james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/StructuredFieldParserTest.java
URL:
http://svn.apache.org/viewvc/james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/StructuredFieldParserTest.java?rev=671327&view=auto
==============================================================================
---
james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/StructuredFieldParserTest.java
(added)
+++
james/mime4j/trunk/src/test/java/org/apache/james/mime4j/field/structured/StructuredFieldParserTest.java
Tue Jun 24 13:03:44 2008
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.james.mime4j.field.structured;
+
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+
+public class StructuredFieldParserTest extends TestCase {
+
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ public void testSimpleField() throws Exception {
+ final String string = "Field Value";
+ assertEquals(string, parse(string));
+ }
+
+ public void testTrim() throws Exception {
+ final String string = "Field Value";
+ assertEquals(string, parse(" \t\r\n" + string + " \t\r\n "));
+ }
+
+ public void testFolding() throws Exception {
+ assertEquals("Field Value", parse("Field \t\r\n Value"));
+ }
+
+ public void testQuotedString() throws Exception {
+ assertEquals("Field Value", parse("\"Field Value\""));
+ assertEquals("Field\t\r\nValue", parse("\"Field\t\r\nValue\""));
+ assertEquals("Field\t\r\nValue", parse("\"Field\t\r\n \t
Value\""));
+ }
+
+ public void testComments() throws Exception {
+ assertEquals("Field", parse("Fi(This is a comment)eld"));
+ assertEquals("Field Value", parse("Fi(This is a comment)eld (A (very
(nested) )comment)Value"));
+ }
+
+ public void testQuotedInComments() throws Exception {
+ assertEquals("Fi(This is a comment)eld", parse("\"Fi(This is a
comment)eld\""));
+ assertEquals("Field Value", parse("Fi(This is a comment)eld (A (very
(nested) )comment)Value"));
+ }
+
+ private String parse(String in) throws Exception {
+ StructuredFieldParser parser = new StructuredFieldParser(new
StringReader(in));
+ return parser.parse();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
