Author: norman
Date: Fri Feb 4 19:18:53 2011
New Revision: 1067263
URL: http://svn.apache.org/viewvc?rev=1067263&view=rev
Log:
Stream big literal to disk if a configurable threshold was hit. This guard
against DOS. The default is 10MB which should make it still very performant by
default as we only need to hit the disk in a few cases. See JAMES-1191
Added:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/AbstractNettyImapRequestLineReader.java
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyStreamImapRequestLineReader.java
Modified:
james/server/trunk/container-spring/src/main/config/james/imapserver.xml
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/IMAPServer.java
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyImapRequestLineReader.java
Modified:
james/server/trunk/container-spring/src/main/config/james/imapserver.xml
URL:
http://svn.apache.org/viewvc/james/server/trunk/container-spring/src/main/config/james/imapserver.xml?rev=1067263&r1=1067262&r2=1067263&view=diff
==============================================================================
--- james/server/trunk/container-spring/src/main/config/james/imapserver.xml
(original)
+++ james/server/trunk/container-spring/src/main/config/james/imapserver.xml
Fri Feb 4 19:18:53 2011
@@ -31,6 +31,9 @@
<!-- Maximal allowed line-length before a BAD response will get returned to
the client -->
<!-- This should be set with caution as a to high value can make the server
a target for DOS! -->
<maxLineLength>65536</maxLineLength>
+
+ <!-- 10MB size limit before we will start to stream to a temporary file -->
+ <inMemorySizeLimit>10485760</inMemorySizeLimit>
<handler>
<connectionLimit> 0 </connectionLimit>
<connectionLimitPerIP> 0 </connectionLimitPerIP>
Added:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/AbstractNettyImapRequestLineReader.java
URL:
http://svn.apache.org/viewvc/james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/AbstractNettyImapRequestLineReader.java?rev=1067263&view=auto
==============================================================================
---
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/AbstractNettyImapRequestLineReader.java
(added)
+++
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/AbstractNettyImapRequestLineReader.java
Fri Feb 4 19:18:53 2011
@@ -0,0 +1,47 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.imapserver.netty;
+
+import org.apache.james.imap.decode.DecodingException;
+import org.apache.james.imap.decode.ImapRequestLineReader;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+
+public abstract class AbstractNettyImapRequestLineReader extends
ImapRequestLineReader{
+ private Channel channel;
+ private ChannelBuffer cRequest =
ChannelBuffers.wrappedBuffer("+\r\n".getBytes());
+ private boolean retry;
+
+ public AbstractNettyImapRequestLineReader(Channel channel, boolean retry) {
+ this.channel = channel;
+ this.retry = retry;
+
+ }
+ /*
+ * (non-Javadoc)
+ * @see
org.apache.james.imap.decode.ImapRequestLineReader#commandContinuationRequest()
+ */
+ protected void commandContinuationRequest() throws DecodingException {
+ // only write the request out if this is not a retry to process the
request..
+
+ if (!retry) channel.write(cRequest);
+ }
+
+}
Modified:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/IMAPServer.java
URL:
http://svn.apache.org/viewvc/james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/IMAPServer.java?rev=1067263&r1=1067262&r2=1067263&view=diff
==============================================================================
---
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/IMAPServer.java
(original)
+++
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/IMAPServer.java
Fri Feb 4 19:18:53 2011
@@ -59,9 +59,14 @@ public class IMAPServer extends Abstract
private boolean compress;
private int maxLineLength;
+
+ private int inMemorySizeLimit;
// Use a big default
public final static int DEFAULT_MAX_LINE_LENGTH = 65536;
+
+ // Use 10MB as default
+ public final static int DEFAULT_IN_MEMORY_SIZE_LIMIT = 10485760;
@Resource(name="imapDecoder")
public void setImapDecoder(ImapDecoder decoder) {
@@ -84,6 +89,7 @@ public class IMAPServer extends Abstract
hello = softwaretype + " Server " + getHelloName() + " is ready.";
compress = configuration.getBoolean("compress", false);
maxLineLength = configuration.getInt("maxLineLength",
DEFAULT_MAX_LINE_LENGTH);
+ inMemorySizeLimit = configuration.getInt("inMemorySizeLimit",
DEFAULT_IN_MEMORY_SIZE_LIMIT);
}
@@ -125,7 +131,7 @@ public class IMAPServer extends Abstract
// Add the text line decoder which limit the max line length,
don't strip the delimiter and use CRLF as delimiter
pipeline.addLast(FRAMER, new
DelimiterBasedFrameDecoder(maxLineLength, false, Delimiters.lineDelimiter()));
- pipeline.addLast(REQUEST_DECODER, new
ImapRequestFrameDecoder(decoder));
+ pipeline.addLast(REQUEST_DECODER, new
ImapRequestFrameDecoder(decoder, inMemorySizeLimit));
if (isSSLSocket()) {
Modified:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java
URL:
http://svn.apache.org/viewvc/james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java?rev=1067263&r1=1067262&r2=1067263&view=diff
==============================================================================
---
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java
(original)
+++
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java
Fri Feb 4 19:18:53 2011
@@ -19,14 +19,25 @@
package org.apache.james.imapserver.netty;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.james.imap.api.ImapMessage;
import org.apache.james.imap.api.process.ImapSession;
import org.apache.james.imap.decode.ImapDecoder;
import org.apache.james.imap.decode.ImapRequestLineReader;
import org.apache.james.protocols.impl.ChannelAttributeSupport;
import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBufferInputStream;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.handler.codec.frame.FrameDecoder;
/**
@@ -35,47 +46,126 @@ import org.jboss.netty.handler.codec.fra
*/
public class ImapRequestFrameDecoder extends FrameDecoder implements
ChannelAttributeSupport{
- private ImapDecoder decoder;
+ private final ImapDecoder decoder;
+ private final int inMemorySizeLimit;
+ private final static String NEEDED_DATA = "NEEDED_DATA";
+ private final static String STORED_DATA = "STORED_DATA";
+ private final static String WRITTEN_DATA = "WRITTEN_DATA";
- public ImapRequestFrameDecoder(ImapDecoder decoder) {
+ public ImapRequestFrameDecoder(ImapDecoder decoder, int inMemorySizeLimit)
{
this.decoder = decoder;
+ this.inMemorySizeLimit = inMemorySizeLimit;
}
+ @Override
+ public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e)
throws Exception {
+ ctx.setAttachment(new HashMap<String, Object>());
+ super.channelOpen(ctx, e);
+ }
+
/*
* (non-Javadoc)
* @see
org.jboss.netty.handler.codec.frame.FrameDecoder#decode(org.jboss.netty.channel.ChannelHandlerContext,
org.jboss.netty.channel.Channel, org.jboss.netty.buffer.ChannelBuffer)
*/
+ @SuppressWarnings("unchecked")
protected Object decode(ChannelHandlerContext ctx, Channel channel,
ChannelBuffer buffer) throws Exception {
buffer.markReaderIndex();
boolean retry = false;
+
+ ImapRequestLineReader reader;
// check if we failed before and if we already know how much data we
need to sucess next run
- Object attachment = ctx.getAttachment();
- if (attachment != null) {
+ Map<String,Object> attachment = (Map<String, Object>)
ctx.getAttachment();
+
+ if (attachment.containsKey(NEEDED_DATA)) {
retry = true;
- int size = (Integer) attachment;
+ int size = (Integer) attachment.get(NEEDED_DATA);
// now see if the buffer hold enough data to process.
if (size !=
NettyImapRequestLineReader.NotEnoughDataException.UNKNOWN_SIZE && size >
buffer.readableBytes()) {
- buffer.resetReaderIndex();
- return null;
+ // check if we have a inMemorySize limit and if so if the
expected size will fit into it
+ if (inMemorySizeLimit > 0 && inMemorySizeLimit < size) {
+
+ // ok seems like it will not fit in the memory limit so we
need to store it in a temporary file
+ final File f;
+ int written;
+
+ // check if we have created a temporary file already or if
we need to create a new one
+ if (attachment.containsKey(STORED_DATA)) {
+ f = (File) attachment.get(STORED_DATA);
+ written = (Integer) attachment.get(WRITTEN_DATA);
+ } else {
+ f = File.createTempFile("imap-literal", ".tmp");
+ attachment.put(STORED_DATA, f);
+ written = 0;
+ attachment.put(WRITTEN_DATA, written);
+
+ }
+
+
+ InputStream bufferIn = null;
+ OutputStream out = null;
+ try {
+ bufferIn = new ChannelBufferInputStream(buffer);
+ out = new FileOutputStream(f, true);
+
+ // write the needed data to the file
+ int i = -1;
+ while (written < size && (i = bufferIn.read()) != -1) {
+ out.write(i);
+ written++;
+ }
+
+
+
+ } finally {
+ if (bufferIn != null) {
+ bufferIn.close();
+ }
+ if (out != null) {
+ out.close();
+ }
+ }
+ // Check if all needed data was streamed to the file.
+ if (written == size) {
+ reader = new NettyStreamImapRequestLineReader(channel,
new FileInputStream(f) {
+ /**
+ * Delete the File on close too
+ */
+ @Override
+ public void close() throws IOException {
+ super.close();
+ f.delete();
+ }
+
+ }, retry);
+ } else {
+ attachment.put(WRITTEN_DATA, written);
+ return null;
+ }
+
+ } else {
+ buffer.resetReaderIndex();
+ return null;
+ }
+
+ } else {
+
+ reader = new NettyImapRequestLineReader(channel, buffer,
retry);
}
+ } else {
+ reader = new NettyImapRequestLineReader(channel, buffer, retry);
}
- try {
-
- ImapRequestLineReader reader = new
NettyImapRequestLineReader(channel, buffer, retry);
+ try {
ImapMessage message = decoder.decode(reader, (ImapSession)
attributes.get(channel));
- // ok no errors found consume the rest of the line
- reader.consumeLine();
-
- ctx.setAttachment(null);
+ attachment.clear();
return message;
} catch (NettyImapRequestLineReader.NotEnoughDataException e) {
// this exception was thrown because we don't have enough data yet
int neededData = e.getNeededSize();
// store the needed data size for later usage
- ctx.setAttachment(neededData);
+ attachment.put(NEEDED_DATA, neededData);
buffer.resetReaderIndex();
return null;
}
Modified:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyImapRequestLineReader.java
URL:
http://svn.apache.org/viewvc/james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyImapRequestLineReader.java?rev=1067263&r1=1067262&r2=1067263&view=diff
==============================================================================
---
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyImapRequestLineReader.java
(original)
+++
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyImapRequestLineReader.java
Fri Feb 4 19:18:53 2011
@@ -26,7 +26,6 @@ import org.apache.james.imap.decode.Imap
import org.apache.james.imap.decode.base.EolInputStream;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBufferInputStream;
-import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
/**
@@ -34,17 +33,14 @@ import org.jboss.netty.channel.Channel;
* see the docs on {@link #nextChar()} and {@link #read(int)} to understand
the special behavior of this implementation
*
*/
-public class NettyImapRequestLineReader extends ImapRequestLineReader{
+public class NettyImapRequestLineReader extends
AbstractNettyImapRequestLineReader{
private ChannelBuffer buffer;
- private Channel channel;
- private ChannelBuffer cRequest =
ChannelBuffers.wrappedBuffer("+\r\n".getBytes());
- private boolean retry;
+ private int read = 0;
public NettyImapRequestLineReader(Channel channel, ChannelBuffer buffer,
boolean retry) {
+ super(channel, retry);
this.buffer = buffer;
- this.channel = channel;
- this.retry = retry;
}
@@ -61,6 +57,7 @@ public class NettyImapRequestLineReader
if (buffer.readable()) {
next = buffer.readByte();
+ read++;
} else {
throw new NotEnoughDataException();
}
@@ -81,7 +78,8 @@ public class NettyImapRequestLineReader
}
// Check if we have enough data
if (size + crlf> buffer.readableBytes()) {
- throw new NotEnoughDataException(size);
+ // ok let us throw a exception which till the decoder how many
more bytes we need
+ throw new NotEnoughDataException(size + read + crlf);
}
// Unset the next char.
@@ -96,16 +94,7 @@ public class NettyImapRequestLineReader
}
}
- /*
- * (non-Javadoc)
- * @see
org.apache.james.imap.decode.ImapRequestLineReader#commandContinuationRequest()
- */
- protected void commandContinuationRequest() throws DecodingException {
- // only write the request out if this is not a retry to process the
request..
-
- if (!retry) channel.write(cRequest);
- }
-
+
/**
* {@link RuntimeException} which will get thrown by {@link
NettyImapRequestLineReader#nextChar()} and {@link
NettyImapRequestLineReader#read(int)} if not
* enough data is readable in the underlying {@link ChannelBuffer}
Added:
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyStreamImapRequestLineReader.java
URL:
http://svn.apache.org/viewvc/james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyStreamImapRequestLineReader.java?rev=1067263&view=auto
==============================================================================
---
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyStreamImapRequestLineReader.java
(added)
+++
james/server/trunk/imapserver/src/main/java/org/apache/james/imapserver/netty/NettyStreamImapRequestLineReader.java
Fri Feb 4 19:18:53 2011
@@ -0,0 +1,101 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.imapserver.netty;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.james.imap.api.display.HumanReadableText;
+import org.apache.james.imap.decode.DecodingException;
+import org.apache.james.imap.decode.base.EolInputStream;
+import org.apache.james.imap.decode.base.FixedLengthInputStream;
+import org.jboss.netty.channel.Channel;
+
+public class NettyStreamImapRequestLineReader extends
AbstractNettyImapRequestLineReader{
+
+ private InputStream in;
+
+ public NettyStreamImapRequestLineReader(Channel channel, InputStream in,
boolean retry) {
+ super(channel, retry);
+ this.in = in;
+ }
+
+ /**
+ * Reads the next character in the current line. This method will continue
+ * to return the same character until the {@link #consume()} method is
+ * called.
+ *
+ * @return The next character TODO: character encoding is variable and
+ * cannot be determine at the token level; this char is not
accurate
+ * reported; should be an octet
+ * @throws DecodingException
+ * If the end-of-stream is reached.
+ */
+ public char nextChar() throws DecodingException {
+ if (!nextSeen) {
+ int next = -1;
+
+ try {
+ next = in.read();
+ } catch (IOException e) {
+ throw new
DecodingException(HumanReadableText.SOCKET_IO_FAILURE,
+ "Error reading from stream.", e);
+ }
+ if (next == -1) {
+ throw new
DecodingException(HumanReadableText.ILLEGAL_ARGUMENTS,
+ "Unexpected end of stream.");
+ }
+
+ nextSeen = true;
+ nextChar = (char) next;
+ }
+ return nextChar;
+ }
+
+
+ /**
+ * Reads and consumes a number of characters from the underlying reader,
+ * filling the char array provided. TODO: remove unnecessary copying of
+ * bits; line reader should maintain an internal ByteBuffer;
+ *
+ * @param holder
+ * A char array which will be filled with chars read from the
+ * underlying reader.
+ * @throws DecodingException
+ * If a char can't be read into each array element.
+ */
+ public InputStream read(int size, boolean extraCRLF) throws
DecodingException {
+
+ // Unset the next char.
+ nextSeen = false;
+ nextChar = 0;
+ FixedLengthInputStream fin = new FixedLengthInputStream(this.in, size);
+ if (extraCRLF) {
+ return new EolInputStream(this, fin);
+ } else {
+ return fin;
+ }
+ }
+
+ public void dispose() throws IOException {
+ in.close();
+ }
+
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
