Hello: We're still trying to work through an issue we have seen in production, only under fairly heavy load - we are seeing truncated headers.
I've listed the code below, but have a specific question: In the 'decodable' method, we return MessageDecoderResult.OK if there is a full HTTP message (ie. headers, blank line, and possibly body if a POST). Note that in the 'decode' method, there is a loop - the comment says that this is for the case where there is more than one request in the buffer (I wrote this comment, but almost 2 years ago). My question - is this wrong? I'm wondering if our issue wrt spurious truncated headers has to do with the fact that one call to 'decodable' might result in two (or more?) calls (within the loop) to ProtocolDecoderOutput.out? Wondering if someone has any bright ideas - it's confusing, as it's tough to reproduce, and we're still trying to get an understanding of why we sometimes get truncated headers (eg. something like 'Content-Len') when decodable has returned true. I've been looking at this code again, and it "seems wrong" that one call to decodable can result in more than one message being decoded, I would rather expect that if decodable returns true, our code is responsible to remove JUST THAT message from the buffer. Sorry I can't be more specific, but any hints/suggestions welcomed. Using mina-M6. Seen on various operating systems. Cheers and thanks. parki... package com.ecobee.communicator.server.mina; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.util.*; import com.ecobee.communicator.server.IServer; import com.ecobee.foundation.Container; import com.ecobee.foundation.net.*; import com.whatevernot.util.Log; import org.apache.mina.core.buffer.IoBuffer; import org.apache.mina.core.session.IoSession; import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.apache.mina.filter.codec.demux.MessageDecoderAdapter; import org.apache.mina.filter.codec.demux.MessageDecoderResult; /** * A request decoder, which handles the parsing of the incoming requests. This * is heavily adapted from the mina example code, but contains a number of changes * since we don't restrict communications to request and response. * @author The Apache MINA Project ([email protected]) * @version $Rev: 593479 $, $Date: 2007-11-09 05:21:35 -0500 (Fri, 09 Nov 2007) $ */ public class HttpRequestDecoder extends MessageDecoderAdapter { final private static String DEFAULT_CONTENT_TYPE = ((IServer) Container.getContext().getBean("server")).getDefaultContentType(); final private static byte[] CONTENT_LENGTH = new String("Content-Length:").getBytes(); final private static int CONTENT_LENGTH_LENGTH = CONTENT_LENGTH.length; final private CharsetDecoder DECODER = Charset.defaultCharset().newDecoder(); private class ParseContext { private int offset; private IoBuffer in; public ParseContext(int offset, IoBuffer in) { this.offset = offset; this.in = in; } } /** * Default constructor, as this object is created dynamically. */ public HttpRequestDecoder() { } /** * @see MessageDecoderAdapter#decodable */ public MessageDecoderResult decodable(IoSession session, IoBuffer in) { try { boolean value = messageComplete(in); if(value) { return MessageDecoderResult.OK; } else { // Return NEED_DATA if the whole header is not read yet. return MessageDecoderResult.NEED_DATA; } } catch (Exception ex) { Log.error(this, "decodable", "Exception decoding HTTP request.", ex); return MessageDecoderResult.NOT_OK; } } /** * @see MessageDecoderAdapter#decode */ public MessageDecoderResult decode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception { ParseContext parseContext = new ParseContext(0, in); // Loop, as it's possible that there is more than one message in the buffer. while(true) { IHttpRequest request = parseRequest(parseContext); if(request == null) break; out.write(request); } return MessageDecoderResult.OK; } private boolean messageComplete(IoBuffer in) throws Exception { // We need at least 4 bytes to make any sense of the request. if(in.remaining() < 4) { return false; } // Loop forwards looking for the line separator 0x0D 0x0A 0x0D 0x0A. int lineSeparatorIndex = findSeparator(in); if(isGet(in)) { // If there is a line separator, we have a valid GET request. return (lineSeparatorIndex != -1); } if(isPost(in)) { // If there is no valid separator, then the POST request is invalid. if(lineSeparatorIndex == -1) { return false; } int last = in.remaining() - 1; for (int i = 0; i <= (last - CONTENT_LENGTH_LENGTH); i++) { boolean found = false; for(int j = 0; j < CONTENT_LENGTH_LENGTH; j++) { if(in.get(i + j) != CONTENT_LENGTH[j]) { found = false; break; } found = true; } if(found) { // Retrieve value from this position till next 0x0D 0x0A. StringBuilder contentLength = new StringBuilder(); for(int j = i + CONTENT_LENGTH.length; j < last; j++) { if (in.get(j) == 0x0D) break; contentLength.append(new String(new byte[] { in.get(j) })); } int intContentLength = Integer.parseInt(contentLength.toString().trim()); // If content-length worth of data has been received then the message is complete. return ((lineSeparatorIndex + 4 + intContentLength) <= in.remaining()); } } } // The message is not complete and we need more data. return false; } private IHttpRequest parseRequest(ParseContext parseContext) throws IOException { int offset = parseContext.offset; IoBuffer in = parseContext.in; // Find the line separator. int separatorIndex = findSeparator(in, offset); // If there is no separator, we need more data. if(separatorIndex == -1) return null; // The end of the message is the separator index + size of the separator. int endOfMessage = separatorIndex + 4; // Update the parse context offset to the beginning of the next message, if any. parseContext.offset = endOfMessage; String contents = in.getString(endOfMessage - offset, DECODER); BufferedReader reader = new BufferedReader(new StringReader(contents)); String firstLine = reader.readLine(); if(firstLine == null) return null; String[] url = firstLine.split(" "); if (url.length < 3) return null; String method = url[0].toUpperCase(); String context = url[1]; String protocol = url[2]; Map<String, String> headers = new HashMap<String, String>(); // Parse up the headers. while(true) { String line = reader.readLine(); if((line == null) || (line.length() == 0)) break; String[] tokens = line.split(": "); if(tokens.length == 2) { headers.put(tokens[0], tokens[1]); } else if(tokens.length == 1) { headers.put(tokens[0], ""); } else { Log.error(this, "parseRequest", "Error setting header: " + line + "; first line: " + firstLine); return null; } } // Determine if the request is compressed and/or encrypted or not. String contentType = headers.get(IHttpRequest.CONTENT_TYPE_HEADER); if(contentType == null) contentType = DEFAULT_CONTENT_TYPE; IHttpRequest request = makeRequest(contentType); request.setMethod(method); request.setContext(context); request.setProtocol(protocol); Iterator<String> iterator = headers.keySet().iterator(); while(iterator.hasNext()) { String key = iterator.next(); String value = headers.get(key); request.setHeader(key, value); } // Parse up any parameters. int idx = context.indexOf('?'); if(idx != -1) { // The context becomes everything up to the '?' character. request.setContext(context.substring(0, idx)); // The parameter list is everything else. String params = url[1].substring(idx + 1); // Split up the parameter list. String[] match = params.split("\\&"); for(String element : match) { String[] tokens = element.split("="); switch(tokens.length) { case 0: request.setParameter(element, ""); break; case 1: request.setParameter(tokens[0], ""); break; default: request.setParameter(tokens[0], tokens[1]); break; } } } // If method 'POST' then read Content-Length worth of data if(request.getMethod() == IHttpRequest.HttpMethod.POST) { int contentLength = request.getContentLength(); byte[] bodyBytes = new byte[contentLength]; in.get(bodyBytes, 0, contentLength); request.setBodyBytes(bodyBytes); } return request; } private int findSeparator(IoBuffer in) { return findSeparator(in, 0); } private int findSeparator(IoBuffer in, int offset) { int last = in.remaining() - 1; for(int i = offset; i <= last - 3; ++i) { if(isSeparator(in, i)) return i; } return -1; } private boolean isSeparator(IoBuffer in, int i) { return ((in.get(i) == (byte) 0x0D && in.get(i + 1) == (byte) 0x0A && in.get(i + 2) == (byte) 0x0D && in.get(i + 3) == (byte) 0x0A)); } private boolean isGet(IoBuffer in) { return (in.get(0) == (byte) 'G') && (in.get(1) == (byte) 'E') && (in.get(2) == (byte) 'T'); } private boolean isPost(IoBuffer in) { return ((in.get(0) == (byte) 'P') && (in.get(1) == (byte) 'O') && (in.get(2) == (byte) 'S') && (in.get(3) == (byte) 'T')); } private IHttpRequest makeRequest(String contentType) { if(IHttpRequest.AES_CONTENT_TYPE.equals(contentType)) { return new AesInboundHttpRequest(); } else if(IHttpRequest.ZLIB_CONTENT_TYPE.equals(contentType)) { return new ZLibInboundHttpRequest(); } else { return new HttpRequest(); } } }
