Repository: johnzon Updated Branches: refs/heads/master 180eb6133 -> d5ba838e4
JOHNZON-125 basic support of auto ajustment for the string sizes Project: http://git-wip-us.apache.org/repos/asf/johnzon/repo Commit: http://git-wip-us.apache.org/repos/asf/johnzon/commit/d5ba838e Tree: http://git-wip-us.apache.org/repos/asf/johnzon/tree/d5ba838e Diff: http://git-wip-us.apache.org/repos/asf/johnzon/diff/d5ba838e Branch: refs/heads/master Commit: d5ba838e476ae59aae883f9f582dad2e70d6a346 Parents: 180eb61 Author: rmannibucau <[email protected]> Authored: Sat Jun 10 20:20:44 2017 +0200 Committer: rmannibucau <[email protected]> Committed: Sat Jun 10 20:20:44 2017 +0200 ---------------------------------------------------------------------- .../core/CommentsJsonStreamParserImpl.java | 15 +++-- .../johnzon/core/JsonParserFactoryImpl.java | 28 +++++---- .../johnzon/core/JsonStreamParserImpl.java | 65 +++++++++++++------- .../org/apache/johnzon/core/OverflowTest.java | 55 +++++++++++++++++ .../jaxrs/ConfigurableJohnzonProvider.java | 4 ++ .../apache/johnzon/jsonb/JohnzonBuilder.java | 2 + .../apache/johnzon/mapper/MapperBuilder.java | 9 +++ 7 files changed, 138 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-core/src/main/java/org/apache/johnzon/core/CommentsJsonStreamParserImpl.java ---------------------------------------------------------------------- diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/CommentsJsonStreamParserImpl.java b/johnzon-core/src/main/java/org/apache/johnzon/core/CommentsJsonStreamParserImpl.java index 39e9f1c..f9d298e 100644 --- a/johnzon-core/src/main/java/org/apache/johnzon/core/CommentsJsonStreamParserImpl.java +++ b/johnzon-core/src/main/java/org/apache/johnzon/core/CommentsJsonStreamParserImpl.java @@ -26,23 +26,26 @@ public class CommentsJsonStreamParserImpl extends JsonStreamParserImpl { public CommentsJsonStreamParserImpl(final InputStream inputStream, final int maxStringLength, final BufferStrategy.BufferProvider<char[]> bufferProvider, - final BufferStrategy.BufferProvider<char[]> valueBuffer) { - super(inputStream, maxStringLength, bufferProvider, valueBuffer); + final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { + super(inputStream, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } public CommentsJsonStreamParserImpl(final InputStream inputStream, final Charset encoding, final int maxStringLength, final BufferStrategy.BufferProvider<char[]> bufferProvider, - final BufferStrategy.BufferProvider<char[]> valueBuffer) { - super(inputStream, encoding, maxStringLength, bufferProvider, valueBuffer); + final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { + super(inputStream, encoding, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } public CommentsJsonStreamParserImpl(final Reader reader, final int maxStringLength, final BufferStrategy.BufferProvider<char[]> bufferProvider, - final BufferStrategy.BufferProvider<char[]> valueBuffer) { - super(reader, maxStringLength, bufferProvider, valueBuffer); + final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { + super(reader, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } @Override http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-core/src/main/java/org/apache/johnzon/core/JsonParserFactoryImpl.java ---------------------------------------------------------------------- diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonParserFactoryImpl.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonParserFactoryImpl.java index d9bfd67..666dae6 100644 --- a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonParserFactoryImpl.java +++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonParserFactoryImpl.java @@ -18,8 +18,10 @@ */ package org.apache.johnzon.core; -import static java.util.Arrays.asList; - +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.stream.JsonParser; +import javax.json.stream.JsonParserFactory; import java.io.InputStream; import java.io.Reader; import java.nio.charset.Charset; @@ -27,15 +29,13 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; -import javax.json.JsonArray; -import javax.json.JsonObject; -import javax.json.stream.JsonParser; -import javax.json.stream.JsonParserFactory; +import static java.util.Arrays.asList; public class JsonParserFactoryImpl extends AbstractJsonFactory implements JsonParserFactory { public static final String MAX_STRING_LENGTH = "org.apache.johnzon.max-string-length"; public static final int DEFAULT_MAX_STRING_LENGTH = Integer.getInteger(MAX_STRING_LENGTH, 10 * 1024 * 1024); //10m + public static final String AUTO_ADJUST_STRING_BUFFER = "org.apache.johnzon.auto-adjust-buffer"; public static final String BUFFER_LENGTH = "org.apache.johnzon.default-char-buffer"; public static final int DEFAULT_BUFFER_LENGTH = Integer.getInteger(BUFFER_LENGTH, 64 * 1024); //64k @@ -43,13 +43,14 @@ public class JsonParserFactoryImpl extends AbstractJsonFactory implements JsonPa public static final boolean DEFAULT_SUPPORTS_COMMENT = Boolean.getBoolean(SUPPORTS_COMMENTS); //default is false; static final Collection<String> SUPPORTED_CONFIG_KEYS = asList( - BUFFER_STRATEGY, MAX_STRING_LENGTH, BUFFER_LENGTH, SUPPORTS_COMMENTS + BUFFER_STRATEGY, MAX_STRING_LENGTH, BUFFER_LENGTH, SUPPORTS_COMMENTS, AUTO_ADJUST_STRING_BUFFER ); private final int maxSize; private final BufferStrategy.BufferProvider<char[]> bufferProvider; private final BufferStrategy.BufferProvider<char[]> valueBufferProvider; private final boolean supportsComments; + private final boolean autoAdjustBuffers; JsonParserFactoryImpl(final Map<String, ?> config) { super(config, SUPPORTED_CONFIG_KEYS, null); @@ -63,30 +64,31 @@ public class JsonParserFactoryImpl extends AbstractJsonFactory implements JsonPa this.bufferProvider = getBufferProvider().newCharProvider(bufferSize); this.valueBufferProvider = getBufferProvider().newCharProvider(maxSize); this.supportsComments = getBool(SUPPORTS_COMMENTS, DEFAULT_SUPPORTS_COMMENT); + this.autoAdjustBuffers = getBool(AUTO_ADJUST_STRING_BUFFER, true); } private JsonStreamParserImpl getDefaultJsonParserImpl(final InputStream in) { if (supportsComments) { - return new CommentsJsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider); + return new CommentsJsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } //UTF Auto detection RFC 4627 - return new JsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider); + return new JsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } private JsonStreamParserImpl getDefaultJsonParserImpl(final InputStream in, final Charset charset) { if (supportsComments) { - return new CommentsJsonStreamParserImpl(in, charset, maxSize, bufferProvider, valueBufferProvider); + return new CommentsJsonStreamParserImpl(in, charset, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } //use provided charset - return new JsonStreamParserImpl(in, charset, maxSize, bufferProvider, valueBufferProvider); + return new JsonStreamParserImpl(in, charset, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } private JsonStreamParserImpl getDefaultJsonParserImpl(final Reader in) { if (supportsComments) { - return new CommentsJsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider); + return new CommentsJsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } //no charset necessary - return new JsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider); + return new JsonStreamParserImpl(in, maxSize, bufferProvider, valueBufferProvider, autoAdjustBuffers); } @Override http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java ---------------------------------------------------------------------- diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java index 44ba2fd..31e9c1b 100644 --- a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java +++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java @@ -31,6 +31,7 @@ import java.util.NoSuchElementException; //This class represents either the Json tokenizer and the Json parser. public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { + private final boolean autoAdjust; //the main buffer where the stream will be buffered private final char[] buffer; @@ -66,7 +67,8 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { //this buffer is used to store current String or Number value in case that //within the value a buffer boundary is crossed or the string contains escaped characters - private final char[] fallBackCopyBuffer; + private char[] fallBackCopyBuffer; + private boolean releaseFallBackCopyBufferLength; private int fallBackCopyBufferLength; // location (line, column, offset) @@ -106,27 +108,31 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { //detect charset according to RFC 4627 public JsonStreamParserImpl(final InputStream inputStream, final int maxStringLength, - final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer) { + final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { - this(inputStream, null, null, maxStringLength, bufferProvider, valueBuffer); + this(inputStream, null, null, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } //use charset provided public JsonStreamParserImpl(final InputStream inputStream, final Charset encoding, final int maxStringLength, - final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer) { + final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { - this(inputStream, null, encoding, maxStringLength, bufferProvider, valueBuffer); + this(inputStream, null, encoding, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } public JsonStreamParserImpl(final Reader reader, final int maxStringLength, final BufferStrategy.BufferProvider<char[]> bufferProvider, - final BufferStrategy.BufferProvider<char[]> valueBuffer) { + final BufferStrategy.BufferProvider<char[]> valueBuffer, final boolean autoAdjust) { - this(null, reader, null, maxStringLength, bufferProvider, valueBuffer); + this(null, reader, null, maxStringLength, bufferProvider, valueBuffer, autoAdjust); } private JsonStreamParserImpl(final InputStream inputStream, final Reader reader, final Charset encoding, final int maxStringLength, - final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer) { + final BufferStrategy.BufferProvider<char[]> bufferProvider, final BufferStrategy.BufferProvider<char[]> valueBuffer, + final boolean autoAdjust) { + this.autoAdjust = autoAdjust; this.maxValueLength = maxStringLength <= 0 ? 8192 : maxStringLength; this.fallBackCopyBuffer = valueBuffer.newBuffer(); this.buffer = bufferProvider.newBuffer(); @@ -154,17 +160,31 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { //copy content between "start" and "end" from buffer to value buffer private void copyCurrentValue() { + final int length = endOfValueInBuffer - startOfValueInBuffer; + if (length > 0) { - if ((endOfValueInBuffer - startOfValueInBuffer) > 0) { - - if ((endOfValueInBuffer - startOfValueInBuffer) > maxValueLength) { + if (length > maxValueLength) { throw tmc(); } - System.arraycopy(buffer, startOfValueInBuffer, fallBackCopyBuffer, fallBackCopyBufferLength, - (endOfValueInBuffer - startOfValueInBuffer)); - fallBackCopyBufferLength += (endOfValueInBuffer - startOfValueInBuffer); + if (fallBackCopyBufferLength >= fallBackCopyBuffer.length - length) { // not good at runtime but handled + if (!autoAdjust) { + throw new ArrayIndexOutOfBoundsException("Buffer too small for such a long string"); + } + final char[] newArray = new char[fallBackCopyBuffer.length + 1024]; // small incr to not explode the mem + // TODO: log to adjust size once? + System.arraycopy(fallBackCopyBuffer, 0, newArray, 0, fallBackCopyBufferLength); + System.arraycopy(buffer, startOfValueInBuffer, newArray, fallBackCopyBufferLength, length); + if (releaseFallBackCopyBufferLength) { + bufferProvider.release(fallBackCopyBuffer); + releaseFallBackCopyBufferLength = false; + } + fallBackCopyBuffer = newArray; + } else { + System.arraycopy(buffer, startOfValueInBuffer, fallBackCopyBuffer, fallBackCopyBufferLength, length); + } + fallBackCopyBufferLength += length; } startOfValueInBuffer = endOfValueInBuffer = -1; @@ -174,9 +194,10 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { public final boolean hasNext() { if (currentStructureElement != null || - (previousEvent != END_ARRAY && previousEvent != END_OBJECT && - previousEvent != VALUE_STRING && previousEvent != VALUE_FALSE && previousEvent != VALUE_TRUE && previousEvent != VALUE_NULL && previousEvent != VALUE_NUMBER) || - previousEvent == 0) { + (previousEvent != END_ARRAY && previousEvent != END_OBJECT && + previousEvent != VALUE_STRING && previousEvent != VALUE_FALSE && previousEvent != VALUE_TRUE && + previousEvent != VALUE_NULL && previousEvent != VALUE_NUMBER) || + previousEvent == 0) { return true; } @@ -432,7 +453,7 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { if (currentStructureElement == null) { currentStructureElement = new StructureElement(null, false); } else { - if(!currentStructureElement.isArray && previousEvent != KEY_SEPARATOR_EVENT) { + if (!currentStructureElement.isArray && previousEvent != KEY_SEPARATOR_EVENT) { throw uexc("Expected :"); } final StructureElement localStructureElement = new StructureElement(currentStructureElement, false); @@ -472,7 +493,7 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { if (currentStructureElement == null) { currentStructureElement = new StructureElement(null, true); } else { - if(!currentStructureElement.isArray && previousEvent != KEY_SEPARATOR_EVENT) { + if (!currentStructureElement.isArray && previousEvent != KEY_SEPARATOR_EVENT) { throw uexc("Expected \""); } final StructureElement localStructureElement = new StructureElement(currentStructureElement, true); @@ -576,7 +597,7 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { bufferPos--; //unread one char } - } while (true); + } while (true); // before this do while(true) it was: // @@ -918,7 +939,9 @@ public class JsonStreamParserImpl implements JsonChars, JohnzonJsonParser { @Override public void close() { bufferProvider.release(buffer); - valueProvider.release(fallBackCopyBuffer); + if (releaseFallBackCopyBufferLength) { + valueProvider.release(fallBackCopyBuffer); + } try { in.close(); http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-core/src/test/java/org/apache/johnzon/core/OverflowTest.java ---------------------------------------------------------------------- diff --git a/johnzon-core/src/test/java/org/apache/johnzon/core/OverflowTest.java b/johnzon-core/src/test/java/org/apache/johnzon/core/OverflowTest.java new file mode 100644 index 0000000..8087fdd --- /dev/null +++ b/johnzon-core/src/test/java/org/apache/johnzon/core/OverflowTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.johnzon.core; + +import org.junit.Test; + +import javax.json.Json; +import javax.json.JsonObject; +import java.io.StringReader; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; + +public class OverflowTest { + @Test + public void ok() { + // normal content size + Json.createReaderFactory(new HashMap<String, Object>() {{ + put(JsonParserFactoryImpl.MAX_STRING_LENGTH, "10"); + put(JsonParserFactoryImpl.BUFFER_LENGTH, "2"); + }}).createReader(new StringReader("{\"a\":\"b\",\n\"another\":\"value\"}")).readObject(); + + // oversized + final JsonObject object = Json.createReaderFactory(new HashMap<String, Object>() {{ + put(JsonParserFactoryImpl.MAX_STRING_LENGTH, "10"); + put(JsonParserFactoryImpl.BUFFER_LENGTH, "2"); + }}).createReader(new StringReader("{\"a\":\"b\",\n\"another\":\"value very long\"}")).readObject(); + assertEquals("value very long", object.getString("another")); + } + + @Test(expected = ArrayIndexOutOfBoundsException.class) + public void ko() { + Json.createReaderFactory(new HashMap<String, Object>() {{ + put(JsonParserFactoryImpl.MAX_STRING_LENGTH, "10"); + put(JsonParserFactoryImpl.BUFFER_LENGTH, "2"); + put(JsonParserFactoryImpl.AUTO_ADJUST_STRING_BUFFER, "false"); + }}).createReader(new StringReader("{\"another\":\"value too long\"}")).readObject(); + } +} http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-jaxrs/src/main/java/org/apache/johnzon/jaxrs/ConfigurableJohnzonProvider.java ---------------------------------------------------------------------- diff --git a/johnzon-jaxrs/src/main/java/org/apache/johnzon/jaxrs/ConfigurableJohnzonProvider.java b/johnzon-jaxrs/src/main/java/org/apache/johnzon/jaxrs/ConfigurableJohnzonProvider.java index 160ee21..4ffa272 100644 --- a/johnzon-jaxrs/src/main/java/org/apache/johnzon/jaxrs/ConfigurableJohnzonProvider.java +++ b/johnzon-jaxrs/src/main/java/org/apache/johnzon/jaxrs/ConfigurableJohnzonProvider.java @@ -228,4 +228,8 @@ public class ConfigurableJohnzonProvider<T> implements MessageBodyWriter<T>, Mes public void setUseBigDecimalForFloats(final boolean useBigDecimalForFloats) { builder.setUseBigDecimalForFloats(useBigDecimalForFloats); } + + public void setAutoAdjustStringBuffers(final boolean autoAdjustStringBuffers) { + builder.setAutoAdjustStringBuffers(autoAdjustStringBuffers); + } } http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-jsonb/src/main/java/org/apache/johnzon/jsonb/JohnzonBuilder.java ---------------------------------------------------------------------- diff --git a/johnzon-jsonb/src/main/java/org/apache/johnzon/jsonb/JohnzonBuilder.java b/johnzon-jsonb/src/main/java/org/apache/johnzon/jsonb/JohnzonBuilder.java index e2b2708..a84cf42 100644 --- a/johnzon-jsonb/src/main/java/org/apache/johnzon/jsonb/JohnzonBuilder.java +++ b/johnzon-jsonb/src/main/java/org/apache/johnzon/jsonb/JohnzonBuilder.java @@ -293,6 +293,8 @@ public class JohnzonBuilder implements JsonbBuilder { builder.setReadAttributeBeforeWrite( config.getProperty("johnzon.readAttributeBeforeWrite").map(Boolean.class::cast).orElse(false)); + builder.setAutoAdjustStringBuffers( + config.getProperty("johnzon.autoAdjustBuffer").map(Boolean.class::cast).orElse(true)); config.getProperty("johnzon.serialize-value-filter") .map(s -> { if (String.class.isInstance(s)) { http://git-wip-us.apache.org/repos/asf/johnzon/blob/d5ba838e/johnzon-mapper/src/main/java/org/apache/johnzon/mapper/MapperBuilder.java ---------------------------------------------------------------------- diff --git a/johnzon-mapper/src/main/java/org/apache/johnzon/mapper/MapperBuilder.java b/johnzon-mapper/src/main/java/org/apache/johnzon/mapper/MapperBuilder.java index ca0244f..36e2e2c 100644 --- a/johnzon-mapper/src/main/java/org/apache/johnzon/mapper/MapperBuilder.java +++ b/johnzon-mapper/src/main/java/org/apache/johnzon/mapper/MapperBuilder.java @@ -107,6 +107,7 @@ public class MapperBuilder { private int maxSize = -1; private int bufferSize = -1; private String bufferStrategy; + private boolean autoAdjustStringBuffers; private Comparator<String> attributeOrder = null; private boolean supportConstructors; private boolean useGetterForCollections; @@ -143,6 +144,9 @@ public class MapperBuilder { if (pretty) { config.put(JsonGenerator.PRETTY_PRINTING, true); } + if (autoAdjustStringBuffers) { + config.put("org.apache.johnzon.auto-adjust-buffer", true); + } if (generatorFactory == null) { generatorFactory = provider.createGeneratorFactory(config); @@ -408,4 +412,9 @@ public class MapperBuilder { this.useBigDecimalForFloats = useBigDecimalForFloats; return this; } + + public MapperBuilder setAutoAdjustStringBuffers(final boolean autoAdjustStringBuffers) { + this.autoAdjustStringBuffers = autoAdjustStringBuffers; + return this; + } }
