Author: markt Date: Tue Jan 13 15:47:56 2015 New Revision: 1651387 URL: http://svn.apache.org/r1651387 Log: Merge AbtsractNioInputBuffer into AbstractInputBuffer now all input buffers support non-blocking reads of HTTP rquest line and headers.
Removed: tomcat/trunk/java/org/apache/coyote/http11/AbstractNioInputBuffer.java Modified: tomcat/trunk/java/org/apache/coyote/http11/AbstractInputBuffer.java tomcat/trunk/java/org/apache/coyote/http11/InternalAprInputBuffer.java tomcat/trunk/java/org/apache/coyote/http11/InternalNio2InputBuffer.java tomcat/trunk/java/org/apache/coyote/http11/InternalNioInputBuffer.java Modified: tomcat/trunk/java/org/apache/coyote/http11/AbstractInputBuffer.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/coyote/http11/AbstractInputBuffer.java?rev=1651387&r1=1651386&r2=1651387&view=diff ============================================================================== --- tomcat/trunk/java/org/apache/coyote/http11/AbstractInputBuffer.java (original) +++ tomcat/trunk/java/org/apache/coyote/http11/AbstractInputBuffer.java Tue Jan 13 15:47:56 2015 @@ -18,11 +18,13 @@ package org.apache.coyote.http11; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import org.apache.coyote.InputBuffer; import org.apache.coyote.Request; import org.apache.juli.logging.Log; import org.apache.tomcat.util.buf.ByteChunk; +import org.apache.tomcat.util.buf.MessageBytes; import org.apache.tomcat.util.http.MimeHeaders; import org.apache.tomcat.util.net.AbstractEndpoint; import org.apache.tomcat.util.net.SocketWrapperBase; @@ -30,7 +32,7 @@ import org.apache.tomcat.util.res.String public abstract class AbstractInputBuffer<S> implements InputBuffer{ - protected static final boolean[] HTTP_TOKEN_CHAR = new boolean[128]; + // -------------------------------------------------------------- Constants /** * The string manager for this package. @@ -39,6 +41,7 @@ public abstract class AbstractInputBuffe StringManager.getManager(Constants.Package); + protected static final boolean[] HTTP_TOKEN_CHAR = new boolean[128]; static { for (int i = 0; i < 128; i++) { if (i < 32) { @@ -164,6 +167,55 @@ public abstract class AbstractInputBuffe protected int lastActiveFilter; + /** + * Parsing state - used for non blocking parsing so that + * when more data arrives, we can pick up where we left off. + */ + private boolean parsingRequestLine; + private int parsingRequestLinePhase = 0; + private boolean parsingRequestLineEol = false; + private int parsingRequestLineStart = 0; + private int parsingRequestLineQPos = -1; + private HeaderParsePosition headerParsePos; + private final HeaderParseData headerData = new HeaderParseData(); + + /** + * Maximum allowed size of the HTTP request line plus headers plus any + * leading blank lines. + */ + protected final int headerBufferSize; + + /** + * Known size of the NioChannel read buffer. + */ + protected int socketReadBufferSize; + + + // ----------------------------------------------------------- Constructors + + public AbstractInputBuffer(Request request, int headerBufferSize) { + + this.request = request; + headers = request.getMimeHeaders(); + + this.headerBufferSize = headerBufferSize; + + filterLibrary = new InputFilter[0]; + activeFilters = new InputFilter[0]; + lastActiveFilter = -1; + + parsingHeader = true; + parsingRequestLine = true; + parsingRequestLinePhase = 0; + parsingRequestLineEol = false; + parsingRequestLineStart = 0; + parsingRequestLineQPos = -1; + headerParsePos = HeaderParsePosition.HEADER_START; + headerData.recycle(); + swallowInput = true; + } + + // ------------------------------------------------------------- Properties /** @@ -229,14 +281,24 @@ public abstract class AbstractInputBuffe } + // ---------------------------------------------------- InputBuffer Methods + /** - * Implementations are expected to call {@link Request#setStartTime(long)} - * as soon as the first byte is read from the request. + * Read some bytes. */ - public abstract boolean parseRequestLine(boolean useAvailableDataOnly) - throws IOException; + @Override + public int doRead(ByteChunk chunk, Request req) + throws IOException { - public abstract boolean parseHeaders() throws IOException; + if (lastActiveFilter == -1) + return inputStreamInputBuffer.doRead(chunk, req); + else + return activeFilters[lastActiveFilter].doRead(chunk,req); + + } + + + // ------------------------------------------------------- Abstract Methods /** * Attempts to read some data into the input buffer. @@ -259,11 +321,8 @@ public abstract class AbstractInputBuffe * connection. */ public void recycle() { - - // Recycle Request object request.recycle(); - // Recycle filters for (int i = 0; i <= lastActiveFilter; i++) { activeFilters[i].recycle(); } @@ -274,6 +333,13 @@ public abstract class AbstractInputBuffe parsingHeader = true; swallowInput = true; + headerParsePos = HeaderParsePosition.HEADER_START; + parsingRequestLine = true; + parsingRequestLinePhase = 0; + parsingRequestLineEol = false; + parsingRequestLineStart = 0; + parsingRequestLineQPos = -1; + headerData.recycle(); } @@ -284,8 +350,6 @@ public abstract class AbstractInputBuffe * to parse the next HTTP request. */ public void nextRequest() { - - // Recycle Request object request.recycle(); // Copy leftover bytes to the beginning of the buffer @@ -312,6 +376,262 @@ public abstract class AbstractInputBuffe parsingHeader = true; swallowInput = true; + headerParsePos = HeaderParsePosition.HEADER_START; + parsingRequestLine = true; + parsingRequestLinePhase = 0; + parsingRequestLineEol = false; + parsingRequestLineStart = 0; + parsingRequestLineQPos = -1; + headerData.recycle(); + } + + + /** + * Read the request line. This function is meant to be used during the + * HTTP request header parsing. Do NOT attempt to read the request body + * using it. + * + * @throws IOException If an exception occurs during the underlying socket + * read operations, or if the given buffer is not big enough to accommodate + * the whole line. + * @return true if data is properly fed; false if no data is available + * immediately and thread should be freed + */ + public boolean parseRequestLine(boolean useAvailableDataOnly) throws IOException { + + //check state + if ( !parsingRequestLine ) return true; + // + // Skipping blank lines + // + if ( parsingRequestLinePhase < 2 ) { + byte chr = 0; + do { + + // Read new bytes if needed + if (pos >= lastValid) { + if (useAvailableDataOnly) { + return false; + } + // Do a simple read with a short timeout + if (!fill(false)) { + // A read is pending, so no longer in initial state + parsingRequestLinePhase = 1; + return false; + } + } + // Set the start time once we start reading data (even if it is + // just skipping blank lines) + if (request.getStartTime() < 0) { + request.setStartTime(System.currentTimeMillis()); + } + chr = buf[pos++]; + } while ((chr == Constants.CR) || (chr == Constants.LF)); + pos--; + + parsingRequestLineStart = pos; + parsingRequestLinePhase = 2; + if (getLog().isDebugEnabled()) { + getLog().debug("Received [" + + new String(buf, pos, lastValid - pos, + StandardCharsets.ISO_8859_1) + + "]"); + } + } + if ( parsingRequestLinePhase == 2 ) { + // + // Reading the method name + // Method name is always US-ASCII + // + boolean space = false; + while (!space) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) //request line parsing + return false; + } + // Spec says no CR or LF in method name + if (buf[pos] == Constants.CR || buf[pos] == Constants.LF) { + throw new IllegalArgumentException( + sm.getString("iib.invalidmethod")); + } + if (buf[pos] == Constants.SP || buf[pos] == Constants.HT) { + space = true; + request.method().setBytes(buf, parsingRequestLineStart, pos - parsingRequestLineStart); + } + pos++; + } + parsingRequestLinePhase = 3; + } + if ( parsingRequestLinePhase == 3 ) { + // Spec says single SP but also be tolerant of multiple and/or HT + boolean space = true; + while (space) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) //request line parsing + return false; + } + if (buf[pos] == Constants.SP || buf[pos] == Constants.HT) { + pos++; + } else { + space = false; + } + } + parsingRequestLineStart = pos; + parsingRequestLinePhase = 4; + } + if (parsingRequestLinePhase == 4) { + // Mark the current buffer position + + int end = 0; + // + // Reading the URI + // + boolean space = false; + while (!space) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) //request line parsing + return false; + } + if (buf[pos] == Constants.SP || buf[pos] == Constants.HT) { + space = true; + end = pos; + } else if ((buf[pos] == Constants.CR) + || (buf[pos] == Constants.LF)) { + // HTTP/0.9 style request + parsingRequestLineEol = true; + space = true; + end = pos; + } else if ((buf[pos] == Constants.QUESTION) + && (parsingRequestLineQPos == -1)) { + parsingRequestLineQPos = pos; + } + pos++; + } + if (parsingRequestLineQPos >= 0) { + request.queryString().setBytes(buf, parsingRequestLineQPos + 1, + end - parsingRequestLineQPos - 1); + request.requestURI().setBytes(buf, parsingRequestLineStart, parsingRequestLineQPos - parsingRequestLineStart); + } else { + request.requestURI().setBytes(buf, parsingRequestLineStart, end - parsingRequestLineStart); + } + parsingRequestLinePhase = 5; + } + if ( parsingRequestLinePhase == 5 ) { + // Spec says single SP but also be tolerant of multiple and/or HT + boolean space = true; + while (space) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) //request line parsing + return false; + } + if (buf[pos] == Constants.SP || buf[pos] == Constants.HT) { + pos++; + } else { + space = false; + } + } + parsingRequestLineStart = pos; + parsingRequestLinePhase = 6; + + // Mark the current buffer position + end = 0; + } + if (parsingRequestLinePhase == 6) { + // + // Reading the protocol + // Protocol is always US-ASCII + // + while (!parsingRequestLineEol) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) //request line parsing + return false; + } + + if (buf[pos] == Constants.CR) { + end = pos; + } else if (buf[pos] == Constants.LF) { + if (end == 0) + end = pos; + parsingRequestLineEol = true; + } + pos++; + } + + if ( (end - parsingRequestLineStart) > 0) { + request.protocol().setBytes(buf, parsingRequestLineStart, end - parsingRequestLineStart); + } else { + request.protocol().setString(""); + } + parsingRequestLine = false; + parsingRequestLinePhase = 0; + parsingRequestLineEol = false; + parsingRequestLineStart = 0; + return true; + } + throw new IllegalStateException("Invalid request line parse phase:"+parsingRequestLinePhase); + } + + + /** + * Parse the HTTP headers. + */ + public boolean parseHeaders() throws IOException { + if (!parsingHeader) { + throw new IllegalStateException( + sm.getString("iib.parseheaders.ise.error")); + } + + HeaderParseStatus status = HeaderParseStatus.HAVE_MORE_HEADERS; + + do { + status = parseHeader(); + // Checking that + // (1) Headers plus request line size does not exceed its limit + // (2) There are enough bytes to avoid expanding the buffer when + // reading body + // Technically, (2) is technical limitation, (1) is logical + // limitation to enforce the meaning of headerBufferSize + // From the way how buf is allocated and how blank lines are being + // read, it should be enough to check (1) only. + if (pos > headerBufferSize + || buf.length - pos < socketReadBufferSize) { + throw new IllegalArgumentException( + sm.getString("iib.requestheadertoolarge.error")); + } + } while ( status == HeaderParseStatus.HAVE_MORE_HEADERS ); + if (status == HeaderParseStatus.DONE) { + parsingHeader = false; + end = pos; + return true; + } else { + return false; + } + } + + + public int getParsingRequestLinePhase() { + return parsingRequestLinePhase; + } + + + protected void expand(int newsize) { + if ( newsize > buf.length ) { + if (parsingHeader) { + throw new IllegalArgumentException( + sm.getString("iib.requestheadertoolarge.error")); + } + // Should not happen + getLog().warn("Expanding buffer size. Old size: " + buf.length + + ", new size: " + newsize, new Exception()); + byte[] tmp = new byte[newsize]; + System.arraycopy(buf,0,tmp,0,buf.length); + buf = tmp; + } } @@ -408,19 +728,308 @@ public abstract class AbstractInputBuffe } - // ---------------------------------------------------- InputBuffer Methods + // --------------------------------------------------------- Private Methods /** - * Read some bytes. + * Parse an HTTP header. + * + * @return false after reading a blank line (which indicates that the + * HTTP header parsing is done */ - @Override - public int doRead(ByteChunk chunk, Request req) + private HeaderParseStatus parseHeader() throws IOException { - if (lastActiveFilter == -1) - return inputStreamInputBuffer.doRead(chunk, req); - else - return activeFilters[lastActiveFilter].doRead(chunk,req); + // + // Check for blank line + // + + byte chr = 0; + while (headerParsePos == HeaderParsePosition.HEADER_START) { + + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) {//parse header + headerParsePos = HeaderParsePosition.HEADER_START; + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + chr = buf[pos]; + + if (chr == Constants.CR) { + // Skip + } else if (chr == Constants.LF) { + pos++; + return HeaderParseStatus.DONE; + } else { + break; + } + + pos++; + + } + + if ( headerParsePos == HeaderParsePosition.HEADER_START ) { + // Mark the current buffer position + headerData.start = pos; + headerParsePos = HeaderParsePosition.HEADER_NAME; + } + + // + // Reading the header name + // Header name is always US-ASCII + // + + while (headerParsePos == HeaderParsePosition.HEADER_NAME) { + + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) { //parse header + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + chr = buf[pos]; + if (chr == Constants.COLON) { + headerParsePos = HeaderParsePosition.HEADER_VALUE_START; + headerData.headerValue = headers.addValue(buf, headerData.start, pos - headerData.start); + pos++; + // Mark the current buffer position + headerData.start = pos; + headerData.realPos = pos; + headerData.lastSignificantChar = pos; + break; + } else if (!HTTP_TOKEN_CHAR[chr]) { + // If a non-token header is detected, skip the line and + // ignore the header + headerData.lastSignificantChar = pos; + return skipLine(); + } + + // chr is next byte of header name. Convert to lowercase. + if ((chr >= Constants.A) && (chr <= Constants.Z)) { + buf[pos] = (byte) (chr - Constants.LC_OFFSET); + } + pos++; + } + + // Skip the line and ignore the header + if (headerParsePos == HeaderParsePosition.HEADER_SKIPLINE) { + return skipLine(); + } + + // + // Reading the header value (which can be spanned over multiple lines) + // + + while (headerParsePos == HeaderParsePosition.HEADER_VALUE_START || + headerParsePos == HeaderParsePosition.HEADER_VALUE || + headerParsePos == HeaderParsePosition.HEADER_MULTI_LINE) { + + if ( headerParsePos == HeaderParsePosition.HEADER_VALUE_START ) { + // Skipping spaces + while (true) { + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) {//parse header + //HEADER_VALUE_START + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + chr = buf[pos]; + if (chr == Constants.SP || chr == Constants.HT) { + pos++; + } else { + headerParsePos = HeaderParsePosition.HEADER_VALUE; + break; + } + } + } + if ( headerParsePos == HeaderParsePosition.HEADER_VALUE ) { + + // Reading bytes until the end of the line + boolean eol = false; + while (!eol) { + + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) {//parse header + //HEADER_VALUE + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + chr = buf[pos]; + if (chr == Constants.CR) { + // Skip + } else if (chr == Constants.LF) { + eol = true; + } else if (chr == Constants.SP || chr == Constants.HT) { + buf[headerData.realPos] = chr; + headerData.realPos++; + } else { + buf[headerData.realPos] = chr; + headerData.realPos++; + headerData.lastSignificantChar = headerData.realPos; + } + + pos++; + } + + // Ignore whitespaces at the end of the line + headerData.realPos = headerData.lastSignificantChar; + + // Checking the first character of the new line. If the character + // is a LWS, then it's a multiline header + headerParsePos = HeaderParsePosition.HEADER_MULTI_LINE; + } + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) {//parse header + //HEADER_MULTI_LINE + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + chr = buf[pos]; + if ( headerParsePos == HeaderParsePosition.HEADER_MULTI_LINE ) { + if ( (chr != Constants.SP) && (chr != Constants.HT)) { + headerParsePos = HeaderParsePosition.HEADER_START; + break; + } else { + // Copying one extra space in the buffer (since there must + // be at least one space inserted between the lines) + buf[headerData.realPos] = chr; + headerData.realPos++; + headerParsePos = HeaderParsePosition.HEADER_VALUE_START; + } + } + } + // Set the header value + headerData.headerValue.setBytes(buf, headerData.start, + headerData.lastSignificantChar - headerData.start); + headerData.recycle(); + return HeaderParseStatus.HAVE_MORE_HEADERS; + } + + + private HeaderParseStatus skipLine() throws IOException { + headerParsePos = HeaderParsePosition.HEADER_SKIPLINE; + boolean eol = false; + + // Reading bytes until the end of the line + while (!eol) { + + // Read new bytes if needed + if (pos >= lastValid) { + if (!fill(false)) { + return HeaderParseStatus.NEED_MORE_DATA; + } + } + + if (buf[pos] == Constants.CR) { + // Skip + } else if (buf[pos] == Constants.LF) { + eol = true; + } else { + headerData.lastSignificantChar = pos; + } + + pos++; + } + if (getLog().isDebugEnabled()) { + getLog().debug(sm.getString("iib.invalidheader", new String(buf, + headerData.start, + headerData.lastSignificantChar - headerData.start + 1, + StandardCharsets.ISO_8859_1))); + } + + headerParsePos = HeaderParsePosition.HEADER_START; + return HeaderParseStatus.HAVE_MORE_HEADERS; + } + + // ----------------------------------------------------------- Inner classes + + private static enum HeaderParseStatus { + DONE, HAVE_MORE_HEADERS, NEED_MORE_DATA + } + + + private static enum HeaderParsePosition { + /** + * Start of a new header. A CRLF here means that there are no more + * headers. Any other character starts a header name. + */ + HEADER_START, + /** + * Reading a header name. All characters of header are HTTP_TOKEN_CHAR. + * Header name is followed by ':'. No whitespace is allowed.<br> + * Any non-HTTP_TOKEN_CHAR (this includes any whitespace) encountered + * before ':' will result in the whole line being ignored. + */ + HEADER_NAME, + /** + * Skipping whitespace before text of header value starts, either on the + * first line of header value (just after ':') or on subsequent lines + * when it is known that subsequent line starts with SP or HT. + */ + HEADER_VALUE_START, + /** + * Reading the header value. We are inside the value. Either on the + * first line or on any subsequent line. We come into this state from + * HEADER_VALUE_START after the first non-SP/non-HT byte is encountered + * on the line. + */ + HEADER_VALUE, + /** + * Before reading a new line of a header. Once the next byte is peeked, + * the state changes without advancing our position. The state becomes + * either HEADER_VALUE_START (if that first byte is SP or HT), or + * HEADER_START (otherwise). + */ + HEADER_MULTI_LINE, + /** + * Reading all bytes until the next CRLF. The line is being ignored. + */ + HEADER_SKIPLINE + } + + + private static class HeaderParseData { + /** + * When parsing header name: first character of the header.<br> + * When skipping broken header line: first character of the header.<br> + * When parsing header value: first character after ':'. + */ + int start = 0; + /** + * When parsing header name: not used (stays as 0).<br> + * When skipping broken header line: not used (stays as 0).<br> + * When parsing header value: starts as the first character after ':'. + * Then is increased as far as more bytes of the header are harvested. + * Bytes from buf[pos] are copied to buf[realPos]. Thus the string from + * [start] to [realPos-1] is the prepared value of the header, with + * whitespaces removed as needed.<br> + */ + int realPos = 0; + /** + * When parsing header name: not used (stays as 0).<br> + * When skipping broken header line: last non-CR/non-LF character.<br> + * When parsing header value: position after the last not-LWS character.<br> + */ + int lastSignificantChar = 0; + /** + * MB that will store the value of the header. It is null while parsing + * header name and is created after the name has been parsed. + */ + MessageBytes headerValue = null; + public void recycle() { + start = 0; + realPos = 0; + lastSignificantChar = 0; + headerValue = null; + } } } Modified: tomcat/trunk/java/org/apache/coyote/http11/InternalAprInputBuffer.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/coyote/http11/InternalAprInputBuffer.java?rev=1651387&r1=1651386&r2=1651387&view=diff ============================================================================== --- tomcat/trunk/java/org/apache/coyote/http11/InternalAprInputBuffer.java (original) +++ tomcat/trunk/java/org/apache/coyote/http11/InternalAprInputBuffer.java Tue Jan 13 15:47:56 2015 @@ -38,7 +38,7 @@ import org.apache.tomcat.util.net.Socket * * @author <a href="mailto:r...@apache.org">Remy Maucherat</a> */ -public class InternalAprInputBuffer extends AbstractNioInputBuffer<Long> { +public class InternalAprInputBuffer extends AbstractInputBuffer<Long> { private static final Log log = LogFactory.getLog(InternalAprInputBuffer.class); Modified: tomcat/trunk/java/org/apache/coyote/http11/InternalNio2InputBuffer.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/coyote/http11/InternalNio2InputBuffer.java?rev=1651387&r1=1651386&r2=1651387&view=diff ============================================================================== --- tomcat/trunk/java/org/apache/coyote/http11/InternalNio2InputBuffer.java (original) +++ tomcat/trunk/java/org/apache/coyote/http11/InternalNio2InputBuffer.java Tue Jan 13 15:47:56 2015 @@ -41,7 +41,7 @@ import org.apache.tomcat.util.net.Socket /** * Output buffer implementation for NIO2. */ -public class InternalNio2InputBuffer extends AbstractNioInputBuffer<Nio2Channel> { +public class InternalNio2InputBuffer extends AbstractInputBuffer<Nio2Channel> { private static final Log log = LogFactory.getLog(InternalNio2InputBuffer.class); Modified: tomcat/trunk/java/org/apache/coyote/http11/InternalNioInputBuffer.java URL: http://svn.apache.org/viewvc/tomcat/trunk/java/org/apache/coyote/http11/InternalNioInputBuffer.java?rev=1651387&r1=1651386&r2=1651387&view=diff ============================================================================== --- tomcat/trunk/java/org/apache/coyote/http11/InternalNioInputBuffer.java (original) +++ tomcat/trunk/java/org/apache/coyote/http11/InternalNioInputBuffer.java Tue Jan 13 15:47:56 2015 @@ -36,7 +36,7 @@ import org.apache.tomcat.util.net.Socket * Implementation of InputBuffer which provides HTTP request header parsing as * well as transfer decoding. */ -public class InternalNioInputBuffer extends AbstractNioInputBuffer<NioChannel> { +public class InternalNioInputBuffer extends AbstractInputBuffer<NioChannel> { private static final Log log = LogFactory.getLog(InternalNioInputBuffer.class); --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tomcat.apache.org For additional commands, e-mail: dev-h...@tomcat.apache.org