Added: tomcat/trunk/java/org/apache/tomcat/util/http/parser/Token.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/tomcat/util/http/parser/Token.java?rev=1300154&view=auto ============================================================================== --- tomcat/trunk/java/org/apache/tomcat/util/http/parser/Token.java (added) +++ tomcat/trunk/java/org/apache/tomcat/util/http/parser/Token.java Tue Mar 13 14:39:24 2012 @@ -0,0 +1,131 @@ +/* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */ +/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ +package org.apache.tomcat.util.http.parser; + +/** + * Describes the input token stream. + */ +@SuppressWarnings("all") // Ignore warnings in generated code +public class Token implements java.io.Serializable { + + /** + * The version identifier for this Serializable class. + * Increment only if the <i>serialized</i> form of the + * class changes. + */ + private static final long serialVersionUID = 1L; + + /** + * An integer that describes the kind of this token. This numbering + * system is determined by JavaCCParser, and a table of these numbers is + * stored in the file ...Constants.java. + */ + public int kind; + + /** The line number of the first character of this Token. */ + public int beginLine; + /** The column number of the first character of this Token. */ + public int beginColumn; + /** The line number of the last character of this Token. */ + public int endLine; + /** The column number of the last character of this Token. */ + public int endColumn; + + /** + * The string image of the token. + */ + public String image; + + /** + * A reference to the next regular (non-special) token from the input + * stream. If this is the last token from the input stream, or if the + * token manager has not read tokens beyond this one, this field is + * set to null. This is true only if this token is also a regular + * token. Otherwise, see below for a description of the contents of + * this field. + */ + public Token next; + + /** + * This field is used to access special tokens that occur prior to this + * token, but after the immediately preceding regular (non-special) token. + * If there are no such special tokens, this field is set to null. + * When there are more than one such special token, this field refers + * to the last of these special tokens, which in turn refers to the next + * previous special token through its specialToken field, and so on + * until the first special token (whose specialToken field is null). + * The next fields of special tokens refer to other special tokens that + * immediately follow it (without an intervening regular token). If there + * is no such token, this field is null. + */ + public Token specialToken; + + /** + * An optional attribute value of the Token. + * Tokens which are not used as syntactic sugar will often contain + * meaningful values that will be used later on by the compiler or + * interpreter. This attribute value is often different from the image. + * Any subclass of Token that actually wants to return a non-null value can + * override this method as appropriate. + */ + public Object getValue() { + return null; + } + + /** + * No-argument constructor + */ + public Token() {} + + /** + * Constructs a new token for the specified Image. + */ + public Token(int kind) + { + this(kind, null); + } + + /** + * Constructs a new token for the specified Image and Kind. + */ + public Token(int kind, String image) + { + this.kind = kind; + this.image = image; + } + + /** + * Returns the image. + */ + public String toString() + { + return image; + } + + /** + * Returns a new Token object, by default. However, if you want, you + * can create and return subclass objects based on the value of ofKind. + * Simply add the cases to the switch for all those special cases. + * For example, if you have a subclass of Token called IDToken that + * you want to create if ofKind is ID, simply add something like : + * + * case MyParserConstants.ID : return new IDToken(ofKind, image); + * + * to the following switch statement. Then you can cast matchedToken + * variable to the appropriate type and use sit in your lexical actions. + */ + public static Token newToken(int ofKind, String image) + { + switch(ofKind) + { + default : return new Token(ofKind, image); + } + } + + public static Token newToken(int ofKind) + { + return newToken(ofKind, null); + } + +} +/* JavaCC - OriginalChecksum=2104130aa3f9189e35a4571dc4c8f0c9 (do not edit this line) */
Propchange: tomcat/trunk/java/org/apache/tomcat/util/http/parser/Token.java ------------------------------------------------------------------------------ svn:eol-style = native Added: tomcat/trunk/java/org/apache/tomcat/util/http/parser/TokenMgrError.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/tomcat/util/http/parser/TokenMgrError.java?rev=1300154&view=auto ============================================================================== --- tomcat/trunk/java/org/apache/tomcat/util/http/parser/TokenMgrError.java (added) +++ tomcat/trunk/java/org/apache/tomcat/util/http/parser/TokenMgrError.java Tue Mar 13 14:39:24 2012 @@ -0,0 +1,148 @@ +/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */ +/* JavaCCOptions: */ +package org.apache.tomcat.util.http.parser; + +/** Token Manager Error. */ +@SuppressWarnings("all") // Ignore warnings in generated code +public class TokenMgrError extends Error +{ + + /** + * The version identifier for this Serializable class. + * Increment only if the <i>serialized</i> form of the + * class changes. + */ + private static final long serialVersionUID = 1L; + + /* + * Ordinals for various reasons why an Error of this type can be thrown. + */ + + /** + * Lexical error occurred. + */ + static final int LEXICAL_ERROR = 0; + + /** + * An attempt was made to create a second instance of a static token manager. + */ + static final int STATIC_LEXER_ERROR = 1; + + /** + * Tried to change to an invalid lexical state. + */ + static final int INVALID_LEXICAL_STATE = 2; + + /** + * Detected (and bailed out of) an infinite loop in the token manager. + */ + static final int LOOP_DETECTED = 3; + + /** + * Indicates the reason why the exception is thrown. It will have + * one of the above 4 values. + */ + int errorCode; + + /** + * Replaces unprintable characters by their escaped (or unicode escaped) + * equivalents in the given string + */ + protected static final String addEscapes(String str) { + StringBuffer retval = new StringBuffer(); + char ch; + for (int i = 0; i < str.length(); i++) { + switch (str.charAt(i)) + { + case 0 : + continue; + case '\b': + retval.append("\\b"); + continue; + case '\t': + retval.append("\\t"); + continue; + case '\n': + retval.append("\\n"); + continue; + case '\f': + retval.append("\\f"); + continue; + case '\r': + retval.append("\\r"); + continue; + case '\"': + retval.append("\\\""); + continue; + case '\'': + retval.append("\\\'"); + continue; + case '\\': + retval.append("\\\\"); + continue; + default: + if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u" + s.substring(s.length() - 4, s.length())); + } else { + retval.append(ch); + } + continue; + } + } + return retval.toString(); + } + + /** + * Returns a detailed message for the Error when it is thrown by the + * token manager to indicate a lexical error. + * Parameters : + * EOFSeen : indicates if EOF caused the lexical error + * curLexState : lexical state in which this error occurred + * errorLine : line number when the error occurred + * errorColumn : column number when the error occurred + * errorAfter : prefix that was seen before this error occurred + * curchar : the offending character + * Note: You can customize the lexical error message by modifying this method. + */ + protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { + return("Lexical error at line " + + errorLine + ", column " + + errorColumn + ". Encountered: " + + (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") + + "after : \"" + addEscapes(errorAfter) + "\""); + } + + /** + * You can also modify the body of this method to customize your error messages. + * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not + * of end-users concern, so you can return something like : + * + * "Internal Error : Please file a bug report .... " + * + * from this method for such cases in the release version of your parser. + */ + public String getMessage() { + return super.getMessage(); + } + + /* + * Constructors of various flavors follow. + */ + + /** No arg constructor. */ + public TokenMgrError() { + } + + /** Constructor with message and reason. */ + public TokenMgrError(String message, int reason) { + super(message); + errorCode = reason; + } + + /** Full Constructor. */ + public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { + this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); + } +} +/* JavaCC - OriginalChecksum=c0e71cb84849413e4aa36c7471643b93 (do not edit this line) */ Propchange: tomcat/trunk/java/org/apache/tomcat/util/http/parser/TokenMgrError.java ------------------------------------------------------------------------------ svn:eol-style = native Added: tomcat/trunk/test/org/apache/tomcat/util/http/parser/TestMediaType.java URL: http://svn.apache.org/viewvc/tomcat/trunk/test/org/apache/tomcat/util/http/parser/TestMediaType.java?rev=1300154&view=auto ============================================================================== --- tomcat/trunk/test/org/apache/tomcat/util/http/parser/TestMediaType.java (added) +++ tomcat/trunk/test/org/apache/tomcat/util/http/parser/TestMediaType.java Tue Mar 13 14:39:24 2012 @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tomcat.util.http.parser; + +import java.io.StringReader; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit tests for {@link HttpParser} focusing on media-type as defined in + * section 3.7 of RFC 2616. + */ +public class TestMediaType { + + // Include whitespace to ensure Parser handles it correctly (it should be + // skipped). + private static final String TYPE = " foo "; + private static final String SUBTYPE = " bar "; + private static final String TYPES = TYPE + "/" + SUBTYPE; + + private static final Parameter PARAM_TOKEN = + new Parameter("a", "b"); + private static final Parameter PARAM_QUOTED = + new Parameter("x", "y"); + private static final Parameter PARAM_EMPTY_QUOTED = + new Parameter("z", "\"\""); + private static final Parameter PARAM_COMPLEX_QUOTED = + new Parameter("w", "\"foo'bar,a=b;x=y\""); + private static final String CHARSET = "UTF-8"; + private static final String WS_CHARSET = " \tUTF-8"; + private static final String CHARSET_WS = "UTF-8 \t"; + // Since this is quoted, it should retain the space at the end + private static final String CHARSET_QUOTED = "\"" + CHARSET_WS + "\""; + private static final Parameter PARAM_CHARSET = + new Parameter("charset", CHARSET); + private static final Parameter PARAM_WS_CHARSET = + new Parameter("charset", WS_CHARSET); + private static final Parameter PARAM_CHARSET_WS = + new Parameter("charset", CHARSET_WS); + private static final Parameter PARAM_CHARSET_QUOTED = + new Parameter("charset", CHARSET_QUOTED); + + + @Test + public void testSimple() throws ParseException { + doTest(); + } + + + @Test + public void testSimpleWithToken() throws ParseException { + doTest(PARAM_TOKEN); + } + + + @Test + public void testSimpleWithQuotedString() throws ParseException { + doTest(PARAM_QUOTED); + } + + + @Test + public void testSimpleWithEmptyQuotedString() throws ParseException { + doTest(PARAM_EMPTY_QUOTED); + } + + + @Test + public void testSimpleWithComplesQuotedString() throws ParseException { + doTest(PARAM_COMPLEX_QUOTED); + } + + + @Test + public void testSimpleWithCharset() throws ParseException { + doTest(PARAM_CHARSET); + } + + + @Test + public void testSimpleWithCharsetWhitespaceBefore() throws ParseException { + doTest(PARAM_WS_CHARSET); + } + + + @Test + public void testSimpleWithCharsetWhitespaceAfter() throws ParseException { + doTest(PARAM_CHARSET_WS); + } + + + @Test + public void testSimpleWithCharsetQuoted() throws ParseException { + doTest(PARAM_CHARSET_QUOTED); + } + + + @Test + public void testSimpleWithAll() throws ParseException { + doTest(PARAM_COMPLEX_QUOTED, PARAM_EMPTY_QUOTED, PARAM_QUOTED, + PARAM_TOKEN, PARAM_CHARSET); + } + + + @Test + public void testCharset() throws ParseException { + StringBuilder sb = new StringBuilder(); + sb.append(TYPES); + sb.append(PARAM_CHARSET); + sb.append(PARAM_TOKEN); + + StringReader sr = new StringReader(sb.toString()); + HttpParser hp = new HttpParser(sr); + AstMediaType m = hp.MediaType(); + + assertEquals(sb.toString().replaceAll(" ", ""), m.toString()); + assertEquals(CHARSET, m.getCharset()); + assertEquals(TYPES.replaceAll(" ", "") + PARAM_TOKEN, + m.toStringNoCharset()); + } + + + @Test + public void testCharsetQuoted() throws ParseException { + StringBuilder sb = new StringBuilder(); + sb.append(TYPES); + sb.append(PARAM_CHARSET_QUOTED); + + StringReader sr = new StringReader(sb.toString()); + HttpParser hp = new HttpParser(sr); + AstMediaType m = hp.MediaType(); + + assertEquals(CHARSET_WS, m.getCharset()); + assertEquals(TYPES.replaceAll(" ", ""), + m.toStringNoCharset()); + } + + + @Test + public void testBug52811() throws ParseException { + String input = "multipart/related;boundary=1_4F50BD36_CDF8C28;" + + "Start=\"<31671603.smil>\";" + + "Type=\"application/smil;charset=UTF-8\""; + + StringReader sr = new StringReader(input); + HttpParser hp = new HttpParser(sr); + AstMediaType m = hp.MediaType(); + + assertTrue(m.children.length == 5); + + // Check the types + assertTrue(m.children[0] instanceof AstType); + assertTrue(m.children[1] instanceof AstSubType); + assertEquals("multipart", m.children[0].toString()); + assertEquals("related", m.children[1].toString()); + + // Check the parameters + AstParameter p = (AstParameter) m.children[2]; + assertTrue(p.children.length == 2); + assertTrue(p.children[0] instanceof AstAttribute); + assertTrue(p.children[1] instanceof AstValue); + assertEquals("boundary", p.children[0].toString()); + assertEquals("1_4F50BD36_CDF8C28", p.children[1].toString()); + + p = (AstParameter) m.children[3]; + assertTrue(p.children.length == 2); + assertTrue(p.children[0] instanceof AstAttribute); + assertTrue(p.children[1] instanceof AstValue); + assertEquals("Start", p.children[0].toString()); + assertEquals("\"<31671603.smil>\"", p.children[1].toString()); + + p = (AstParameter) m.children[4]; + assertTrue(p.children.length == 2); + assertTrue(p.children[0] instanceof AstAttribute); + assertTrue(p.children[1] instanceof AstValue); + assertEquals("Type", p.children[0].toString()); + assertEquals("\"application/smil;charset=UTF-8\"", + p.children[1].toString()); + + assertEquals(input, m.toString()); + assertEquals(input, m.toStringNoCharset()); + assertNull(m.getCharset()); + } + + + private void doTest(Parameter... parameters) throws ParseException { + StringBuilder sb = new StringBuilder(); + sb.append(TYPES); + for (Parameter p : parameters) { + sb.append(p.toString()); + } + + StringReader sr = new StringReader(sb.toString()); + HttpParser hp = new HttpParser(sr); + AstMediaType m = hp.MediaType(); + + // Check all expected children are present + assertTrue(m.children.length == 2 + parameters.length); + + // Check the types + assertTrue(m.children[0] instanceof AstType); + assertTrue(m.children[1] instanceof AstSubType); + assertEquals(TYPE.trim(), m.children[0].toString()); + assertEquals(SUBTYPE.trim(), m.children[1].toString()); + + // Check the parameters + for (int i = 0; i < parameters.length; i++) { + assertTrue(m.children[i + 2] instanceof AstParameter); + AstParameter p = (AstParameter) m.children[i + 2]; + assertTrue(p.children.length == 2); + assertTrue(p.children[0] instanceof AstAttribute); + assertTrue(p.children[1] instanceof AstValue); + assertEquals(parameters[i].getName().trim(), p.children[0].toString()); + assertEquals(parameters[i].getValue().trim(), p.children[1].toString()); + } + } + + + private static class Parameter { + private final String name; + private final String value; + + public Parameter(String name,String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(";"); + sb.append(name); + sb.append("="); + sb.append(value); + return sb.toString(); + } + } +} Propchange: tomcat/trunk/test/org/apache/tomcat/util/http/parser/TestMediaType.java ------------------------------------------------------------------------------ svn:eol-style = native --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org For additional commands, e-mail: dev-h...@tomcat.apache.org