This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5d02b03854911d7fb889b078f0ada37b282261f9 Author: Michael Blow <[email protected]> AuthorDate: Thu May 6 16:36:30 2021 -0400 [NO ISSUE][*DB] Enable large parsing resources to be freed on memory pressure Change-Id: Ieaca566f765a5d3531baff8128d6ff03c95956e2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11365 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- .../{ObjectPool.java => AbstractObjectPool.java} | 28 +++++++--------- .../external/parser/jackson/IObjectPool.java | 25 +++++--------- .../external/parser/jackson/ObjectPool.java | 34 +++++-------------- .../external/parser/jackson/ParserContext.java | 8 ++--- .../{ObjectPool.java => SoftObjectPool.java} | 38 +++++++--------------- .../apache/hyracks/util/string/UTF8StringUtil.java | 11 ++++--- .../hyracks/util/string/UTF8StringWriter.java | 3 +- 7 files changed, 52 insertions(+), 95 deletions(-) diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java similarity index 75% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java index 8945e71..ef75688 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java @@ -27,39 +27,35 @@ import org.apache.asterix.om.util.container.IObjectFactory; * Object pool for DFS traversal mode, which allows to recycle objects * as soon as it is not needed. */ -public class ObjectPool<E, T> { +public abstract class AbstractObjectPool<E, T, Q> implements IObjectPool<E> { private final IObjectFactory<E, T> objectFactory; - private final Queue<E> recycledObjects; - private final T element; + private final Queue<Q> recycledObjects; + private final T param; - public ObjectPool() { - this(null, null); - } - - public ObjectPool(IObjectFactory<E, T> objectFactory) { - this(objectFactory, null); - } - - public ObjectPool(IObjectFactory<E, T> objectFactory, T element) { + protected AbstractObjectPool(IObjectFactory<E, T> objectFactory, T param) { this.objectFactory = objectFactory; recycledObjects = new ArrayDeque<>(); - this.element = element; + this.param = param; } public E getInstance() { - E instance = recycledObjects.poll(); + E instance = unwrap(recycledObjects.poll()); if (objectFactory != null && instance == null) { - instance = objectFactory.create(element); + instance = objectFactory.create(param); } return instance; } public void recycle(E object) { if (object != null) { - recycledObjects.add(object); + recycledObjects.add(wrap(object)); } } + protected abstract E unwrap(Q element); + + protected abstract Q wrap(E element); + @Override public String toString() { return recycledObjects.toString(); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java similarity index 57% copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java index 563f865..a2990ec 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java @@ -16,23 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.util.string; +package org.apache.asterix.external.parser.jackson; -import java.io.DataOutput; -import java.io.IOException; -import java.io.Serializable; - -public class UTF8StringWriter implements Serializable { - - private static final long serialVersionUID = 1L; - transient byte[] tempBytes; - - public final void writeUTF8(CharSequence str, DataOutput out) throws IOException { - UTF8StringUtil.writeUTF8(str, out, this); - } - - public final void writeUTF8(char[] buffer, int start, int length, DataOutput out) throws IOException { - UTF8StringUtil.writeUTF8(buffer, start, length, out, this); - } +/** + * Object pool for DFS traversal mode, which allows to recycle objects + * as soon as it is not needed. + */ +public interface IObjectPool<E> { + E getInstance(); + void recycle(E object); } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java index 8945e71..864d9a9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java @@ -18,20 +18,13 @@ */ package org.apache.asterix.external.parser.jackson; -import java.util.ArrayDeque; -import java.util.Queue; - import org.apache.asterix.om.util.container.IObjectFactory; /** * Object pool for DFS traversal mode, which allows to recycle objects * as soon as it is not needed. */ -public class ObjectPool<E, T> { - private final IObjectFactory<E, T> objectFactory; - private final Queue<E> recycledObjects; - private final T element; - +public class ObjectPool<E, T> extends AbstractObjectPool<E, T, E> { public ObjectPool() { this(null, null); } @@ -40,28 +33,17 @@ public class ObjectPool<E, T> { this(objectFactory, null); } - public ObjectPool(IObjectFactory<E, T> objectFactory, T element) { - this.objectFactory = objectFactory; - recycledObjects = new ArrayDeque<>(); - this.element = element; + public ObjectPool(IObjectFactory<E, T> objectFactory, T param) { + super(objectFactory, param); } - public E getInstance() { - E instance = recycledObjects.poll(); - if (objectFactory != null && instance == null) { - instance = objectFactory.create(element); - } - return instance; - } - - public void recycle(E object) { - if (object != null) { - recycledObjects.add(object); - } + @Override + protected E unwrap(E wrapped) { + return wrapped; } @Override - public String toString() { - return recycledObjects.toString(); + protected E wrap(E element) { + return element; } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java index d0b79b1..ef9ff08 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java @@ -51,7 +51,7 @@ import org.apache.hyracks.util.string.UTF8StringWriter; public class ParserContext { private static final int SERIALIZED_FIELDNAME_MAP_MAX_SIZE = 128; - private final ObjectPool<IARecordBuilder, ATypeTag> objectBuilderPool; + private final IObjectPool<IARecordBuilder> objectBuilderPool; private final ObjectPool<IAsterixListBuilder, ATypeTag> arrayBuilderPool; /** @@ -61,7 +61,7 @@ public class ParserContext { * <p> * Scalar value 5 is written 4 times in tempBuffer("d") then tempBuffer("c") ... tempBuffer("a") */ - private final ObjectPool<IMutableValueStorage, ATypeTag> tempBufferPool; + private final IObjectPool<IMutableValueStorage> tempBufferPool; private final ObjectPool<BitSet, Void> nullBitmapPool; private final Map<String, IMutableValueStorage> serializedFieldNames; private final ISerializerDeserializer<AString> stringSerDe; @@ -76,9 +76,9 @@ public class ParserContext { @SuppressWarnings("unchecked") public ParserContext(boolean allocateModfiedUTF8Writer) { - objectBuilderPool = new ObjectPool<>(new RecordBuilderFactory()); + objectBuilderPool = new SoftObjectPool<>(new RecordBuilderFactory()); arrayBuilderPool = new ObjectPool<>(new ListBuilderFactory(), ATypeTag.ARRAY); - tempBufferPool = new ObjectPool<>(new AbvsBuilderFactory()); + tempBufferPool = new SoftObjectPool<>(new AbvsBuilderFactory()); nullBitmapPool = new ObjectPool<>(); serializedFieldNames = new LRUMap<>(SERIALIZED_FIELDNAME_MAP_MAX_SIZE); stringSerDe = SerializerDeserializerProvider.INSTANCE.getAStringSerializerDeserializer(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java similarity index 55% copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java index 8945e71..91655db 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java @@ -18,8 +18,7 @@ */ package org.apache.asterix.external.parser.jackson; -import java.util.ArrayDeque; -import java.util.Queue; +import java.lang.ref.SoftReference; import org.apache.asterix.om.util.container.IObjectFactory; @@ -27,41 +26,26 @@ import org.apache.asterix.om.util.container.IObjectFactory; * Object pool for DFS traversal mode, which allows to recycle objects * as soon as it is not needed. */ -public class ObjectPool<E, T> { - private final IObjectFactory<E, T> objectFactory; - private final Queue<E> recycledObjects; - private final T element; - - public ObjectPool() { +public class SoftObjectPool<E, T> extends AbstractObjectPool<E, T, SoftReference<E>> { + public SoftObjectPool() { this(null, null); } - public ObjectPool(IObjectFactory<E, T> objectFactory) { + public SoftObjectPool(IObjectFactory<E, T> objectFactory) { this(objectFactory, null); } - public ObjectPool(IObjectFactory<E, T> objectFactory, T element) { - this.objectFactory = objectFactory; - recycledObjects = new ArrayDeque<>(); - this.element = element; - } - - public E getInstance() { - E instance = recycledObjects.poll(); - if (objectFactory != null && instance == null) { - instance = objectFactory.create(element); - } - return instance; + public SoftObjectPool(IObjectFactory<E, T> objectFactory, T element) { + super(objectFactory, element); } - public void recycle(E object) { - if (object != null) { - recycledObjects.add(object); - } + @Override + protected E unwrap(SoftReference<E> wrapped) { + return wrapped == null ? null : wrapped.get(); } @Override - public String toString() { - return recycledObjects.toString(); + protected SoftReference<E> wrap(E element) { + return new SoftReference<>(element); } } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java index f96ed72..38de4ac 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,6 +25,7 @@ import java.io.EOFException; import java.io.IOException; import java.io.OutputStream; import java.io.UTFDataFormatException; +import java.lang.ref.SoftReference; import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder; @@ -690,10 +691,12 @@ public class UTF8StringUtil { if (writer == null) { tempBytes = new byte[utflen + 5]; } else { - if (writer.tempBytes == null || writer.tempBytes.length < utflen + 5) { - writer.tempBytes = new byte[utflen + 5]; + byte[] writerTempBytes = writer.tempBytesRef != null ? writer.tempBytesRef.get() : null; + if (writerTempBytes == null || writerTempBytes.length < utflen + 5) { + writerTempBytes = new byte[utflen + 5]; + writer.tempBytesRef = new SoftReference<>(writerTempBytes); } - tempBytes = writer.tempBytes; + tempBytes = writerTempBytes; } return tempBytes; } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java index 563f865..a0cc7d0 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java @@ -21,11 +21,12 @@ package org.apache.hyracks.util.string; import java.io.DataOutput; import java.io.IOException; import java.io.Serializable; +import java.lang.ref.SoftReference; public class UTF8StringWriter implements Serializable { private static final long serialVersionUID = 1L; - transient byte[] tempBytes; + transient SoftReference<byte[]> tempBytesRef; public final void writeUTF8(CharSequence str, DataOutput out) throws IOException { UTF8StringUtil.writeUTF8(str, out, this);
