[CARBONDATA-1488] JVM crashes when unsafe columnpage is enabled Fixed capacity size calculation when column value size is larger than initial capacity in this PR.
This closes #1370 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9c24c50 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9c24c50 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9c24c50 Branch: refs/heads/streaming_ingest Commit: e9c24c50536e7700fe4ce468114820cb21942ee3 Parents: 36ceb59 Author: Ravindra Pesala <[email protected]> Authored: Tue Sep 19 16:24:35 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Tue Sep 19 20:13:32 2017 +0800 ---------------------------------------------------------------------- .../core/datastore/page/UnsafeVarLengthColumnPage.java | 12 ------------ .../core/datastore/page/VarLengthColumnPageBase.java | 2 +- .../vectorreader/VectorizedCarbonRecordReader.java | 4 ++-- 3 files changed, 3 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java index c9737a4..9d6e161 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java @@ -42,18 +42,6 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { baseOffset = memoryBlock.getBaseOffset(); } - /** - * create a page with initial capacity - */ - UnsafeVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize, - int capacity) throws MemoryException { - super(columnSpec, dataType, pageSize); - this.capacity = capacity; - memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, (long)(capacity)); - baseAddress = memoryBlock.getBaseObject(); - baseOffset = memoryBlock.getBaseOffset(); - } - @Override public void freeMemory() { if (memoryBlock != null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java index 83e6ef0..276b7ff 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java @@ -363,7 +363,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { */ protected void ensureMemory(int requestSize) throws MemoryException { if (totalLength + requestSize > capacity) { - int newSize = 2 * capacity; + int newSize = Math.max(2 * capacity, totalLength + requestSize); MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize); CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset, newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity); http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9c24c50/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java index 5356ad5..e1946a9 100644 --- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java +++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java @@ -72,9 +72,9 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> { private boolean returnColumnarBatch; /** - * The default config on whether columnarBatch should be offheap. + * The default config on whether columnarBatch should be onheap. */ - private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.OFF_HEAP; + private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.ON_HEAP; private QueryModel queryModel;
