Repository: carbondata Updated Branches: refs/heads/master 63b930c89 -> 892594743
[CARBONDATA-2735] Fixed Performance issue for complex array data type when number of elements in array is more GC issue due to repeated arraylist resize, is fixed This closes #2487 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89259474 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89259474 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89259474 Branch: refs/heads/master Commit: 89259474374eda6ba8754b213eb401b70c093f3e Parents: 63b930c Author: kumarvishal09 <[email protected]> Authored: Wed Jul 11 15:24:08 2018 +0530 Committer: Venkata Ramana G <[email protected]> Committed: Thu Jul 19 11:37:00 2018 +0530 ---------------------------------------------------------------------- .../core/datastore/page/ColumnPage.java | 6 +- .../datastore/page/SafeDecimalColumnPage.java | 1 + .../datastore/page/SafeFixLengthColumnPage.java | 6 +- .../datastore/page/SafeVarLengthColumnPage.java | 1 + .../datastore/page/UnsafeDecimalColumnPage.java | 13 +-- .../page/UnsafeFixLengthColumnPage.java | 27 ++++- .../page/UnsafeVarLengthColumnPage.java | 17 +-- .../datastore/page/VarLengthColumnPageBase.java | 106 +++++++++++-------- 8 files changed, 113 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java index 8745545..ea250cf 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java @@ -49,7 +49,7 @@ import static org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT_INT; public abstract class ColumnPage { // number of row in this page - protected final int pageSize; + protected int pageSize; // data type of the page storage protected final DataType dataType; @@ -832,4 +832,8 @@ public abstract class ColumnPage { public PageLevelDictionary getColumnPageDictionary() { throw new UnsupportedOperationException("Operation not supported"); } + + public int getActualRowCount() { + throw new UnsupportedOperationException("Operation not supported"); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java index 0e0ba85..c220fa4 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java @@ -218,5 +218,6 @@ public class SafeDecimalColumnPage extends DecimalColumnPage { @Override public void freeMemory() { byteArrayData = null; + super.freeMemory(); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java index 8f0d934..74bb3fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java @@ -388,7 +388,7 @@ public class SafeFixLengthColumnPage extends ColumnPage { } } - protected void ensureArraySize(int requestSize, DataType dataType) { + private void ensureArraySize(int requestSize, DataType dataType) { if (dataType == DataTypes.BYTE) { if (requestSize >= byteData.length) { byte[] newArray = new byte[arrayElementCount + 16]; @@ -436,4 +436,8 @@ public class SafeFixLengthColumnPage extends ColumnPage { "not support value conversion on " + dataType + " page"); } } + public int getActualRowCount() { + return arrayElementCount; + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java index c2eb40c..274b8a7 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java @@ -40,6 +40,7 @@ public class SafeVarLengthColumnPage extends VarLengthColumnPageBase { @Override public void freeMemory() { byteArrayData = null; + super.freeMemory(); } @Override http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java index 7449da6..e4a949a 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java @@ -119,6 +119,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage { memoryBlock = null; baseAddress = null; baseOffset = 0; + super.freeMemory(); } } @@ -168,7 +169,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage { throw new RuntimeException(e); } CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress, - baseOffset + rowOffset.get(rowId), length); + baseOffset + rowOffset.getInt(rowId), length); } @Override @@ -193,9 +194,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage { @Override public byte[] getBytes(int rowId) { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); byte[] bytes = new byte[length]; - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); return bytes; } @@ -242,9 +243,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage { } else if (dataType == DataTypes.LONG) { value = getLong(rowId); } else { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); byte[] bytes = new byte[length]; - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), bytes, + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); return decimalConverter.getDecimal(bytes); } @@ -253,7 +254,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage { @Override void copyBytes(int rowId, byte[] dest, int destOffset, int length) { - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), dest, + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java index a4cea5d..c5805d5 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java @@ -110,6 +110,12 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { } } + private void updatePageSize(int rowId) { + if (pageSize < rowId) { + pageSize = rowId; + } + } + @Override public void putByte(int rowId, byte value) { try { @@ -120,6 +126,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long offset = ((long)rowId) << byteBits; CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value); totalLength += ByteUtil.SIZEOF_BYTE; + updatePageSize(rowId); } @@ -134,6 +141,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long offset = ((long)rowId) << shortBits; CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value); totalLength += ByteUtil.SIZEOF_SHORT; + updatePageSize(rowId); } @Override @@ -149,6 +157,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]); CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]); totalLength += ByteUtil.SIZEOF_SHORT_INT; + updatePageSize(rowId); } @Override @@ -161,6 +170,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long offset = ((long)rowId) << intBits; CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value); totalLength += ByteUtil.SIZEOF_INT; + updatePageSize(rowId); } @Override @@ -173,6 +183,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long offset = ((long)rowId) << longBits; CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value); totalLength += ByteUtil.SIZEOF_LONG; + updatePageSize(rowId); } @Override @@ -185,6 +196,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { long offset = ((long)rowId) << doubleBits; CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value); totalLength += ByteUtil.SIZEOF_DOUBLE; + updatePageSize(rowId); } @Override @@ -194,6 +206,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { CarbonUnsafe.getUnsafe() .copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(), baseOffset + offset, bytes.length); + updatePageSize(rowId); } @Override @@ -274,7 +287,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public byte[] getBytePage() { - byte[] data = new byte[getPageSize()]; + byte[] data = new byte[getEndLoop()]; for (long i = 0; i < data.length; i++) { long offset = i << byteBits; data[(int)i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset); @@ -284,7 +297,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public short[] getShortPage() { - short[] data = new short[getPageSize()]; + short[] data = new short[getEndLoop()]; for (long i = 0; i < data.length; i++) { long offset = i << shortBits; data[(int)i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset); @@ -294,7 +307,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public byte[] getShortIntPage() { - byte[] data = new byte[pageSize * 3]; + byte[] data = new byte[getEndLoop() * 3]; CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset, data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length); return data; @@ -302,7 +315,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public int[] getIntPage() { - int[] data = new int[getPageSize()]; + int[] data = new int[getEndLoop()]; for (long i = 0; i < data.length; i++) { long offset = i << intBits; data[(int)i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset); @@ -312,7 +325,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { @Override public long[] getLongPage() { - long[] data = new long[getPageSize()]; + long[] data = new long[getEndLoop()]; for (long i = 0; i < data.length; i++) { long offset = i << longBits; data[(int)i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset); @@ -514,4 +527,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage { capacity = newSize; } } + + public int getActualRowCount() { + return getEndLoop(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java index f60e505..ae57dcd 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java @@ -49,6 +49,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { memoryBlock = null; baseAddress = null; baseOffset = 0; + super.freeMemory(); } } @@ -65,7 +66,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { throw new RuntimeException(e); } CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, - baseAddress, baseOffset + rowOffset.get(rowId), length); + baseAddress, baseOffset + rowOffset.getInt(rowId), length); } @Override @@ -89,20 +90,20 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { @Override public byte[] getBytes(int rowId) { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); byte[] bytes = new byte[length]; - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); return bytes; } @Override public byte[][] getByteArrayPage() { - byte[][] bytes = new byte[rowOffset.size() - 1][]; - for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + byte[][] bytes = new byte[rowOffset.getActualRowCount() - 1][]; + for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); byte[] rowData = new byte[length]; - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length); bytes[rowId] = rowData; } @@ -111,7 +112,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase { @Override void copyBytes(int rowId, byte[] dest, int destOffset, int length) { - CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), + CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java index fa163bc..55877c3 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java @@ -18,10 +18,10 @@ package org.apache.carbondata.core.datastore.page; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; +import org.apache.carbondata.core.datastore.ColumnType; import org.apache.carbondata.core.datastore.TableSpec; import org.apache.carbondata.core.memory.CarbonUnsafe; import org.apache.carbondata.core.memory.MemoryBlock; @@ -53,7 +53,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { Object baseAddress; // the offset of row in the unsafe memory, its size is pageSize + 1 - List<Integer> rowOffset; + ColumnPage rowOffset; // the length of bytes added in the page int totalLength; @@ -66,7 +66,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { VarLengthColumnPageBase(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize) { super(columnSpec, dataType, pageSize); - rowOffset = new ArrayList<>(); + TableSpec.ColumnSpec spec = TableSpec.ColumnSpec + .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); + try { + rowOffset = + ColumnPage.newPage(spec, DataTypes.INT, pageSize); + } catch (MemoryException e) { + throw new RuntimeException(e); + } totalLength = 0; } @@ -142,14 +149,19 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { private static ColumnPage getDecimalColumnPage(TableSpec.ColumnSpec columnSpec, byte[] lvEncodedBytes, int size) throws MemoryException { - List<Integer> rowOffset = new ArrayList<>(); + TableSpec.ColumnSpec spec = TableSpec.ColumnSpec + .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); + ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT, + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int offset; int rowId = 0; + int counter = 0; for (offset = 0; offset < lvEncodedBytes.length; offset += size) { - rowOffset.add(offset); + rowOffset.putInt(counter, offset); rowId++; + counter++; } - rowOffset.add(offset); + rowOffset.putInt(counter, offset); VarLengthColumnPageBase page; if (unsafe) { @@ -160,10 +172,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { // set total length and rowOffset in page page.totalLength = offset; - page.rowOffset = new ArrayList<>(); - for (int i = 0; i < rowOffset.size(); i++) { - page.rowOffset.add(rowOffset.get(i)); - } + page.rowOffset = rowOffset; for (int i = 0; i < rowId; i++) { page.putBytes(i, lvEncodedBytes, i * size, size); } @@ -175,24 +184,26 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; - List<Integer> rowOffset = new ArrayList<>(); - List<Integer> rowLength = new ArrayList<>(); + TableSpec.ColumnSpec spec = TableSpec.ColumnSpec + .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); + ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT, + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; - + int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset); - rowOffset.add(offset); - rowLength.add(length); + rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; + counter++; } - rowOffset.add(offset); + rowOffset.putInt(counter, offset); VarLengthColumnPageBase page = getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, - rowLength, offset); + offset); return page; } @@ -201,31 +212,33 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { throws MemoryException { // extract length and data, set them to rowOffset and unsafe memory correspondingly int rowId = 0; - List<Integer> rowOffset = new ArrayList<>(); - List<Integer> rowLength = new ArrayList<>(); + TableSpec.ColumnSpec spec = TableSpec.ColumnSpec + .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE); + ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT, + CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT); int length; int offset; int lvEncodedOffset = 0; - + int counter = 0; // extract Length field in input and calculate total length for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) { length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset); - rowOffset.add(offset); - rowLength.add(length); + rowOffset.putInt(counter, offset); lvEncodedOffset += lvLength + length; rowId++; + counter++; } - rowOffset.add(offset); + rowOffset.putInt(counter, offset); VarLengthColumnPageBase page = getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset, - rowLength, offset); + offset); return page; } private static VarLengthColumnPageBase getVarLengthColumnPage(TableSpec.ColumnSpec columnSpec, - byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, List<Integer> rowOffset, - List<Integer> rowLength, int offset) throws MemoryException { + byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, ColumnPage rowOffset, + int offset) throws MemoryException { int lvEncodedOffset; int length; int numRows = rowId; @@ -240,15 +253,12 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { // set total length and rowOffset in page page.totalLength = offset; - page.rowOffset = new ArrayList<>(); - for (int i = 0; i < rowOffset.size(); i++) { - page.rowOffset.add(rowOffset.get(i)); - } + page.rowOffset = rowOffset; // set data in page lvEncodedOffset = 0; for (int i = 0; i < numRows; i++) { - length = rowLength.get(i); + length = rowOffset.getInt(i + 1) - rowOffset.getInt(i); page.putBytes(i, lvEncodedBytes, lvEncodedOffset + lvLength, length); lvEncodedOffset += lvLength + length; } @@ -296,9 +306,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { + " exceed this limit at rowId " + rowId); } if (rowId == 0) { - rowOffset.add(0); + rowOffset.putInt(0, 0); } - rowOffset.add(rowOffset.get(rowId) + bytes.length); + rowOffset.putInt(rowId + 1, rowOffset.getInt(rowId) + bytes.length); putBytesAtRow(rowId, bytes); totalLength += bytes.length; } @@ -379,7 +389,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { int offset = 0; byte[] data = new byte[totalLength]; for (int rowId = 0; rowId < pageSize; rowId++) { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); copyBytes(rowId, data, offset, length); offset += length; } @@ -395,9 +405,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { public byte[] getLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; - byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 4)]; - for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) { - int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId); + byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 4)]; + for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { + int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId); ByteUtil.setInt(data, offset, length); copyBytes(rowId, data, offset + 4, length); offset += 4 + length; @@ -408,9 +418,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { @Override public byte[] getComplexChildrenLVFlattenedBytePage() throws IOException { // output LV encoded byte array int offset = 0; - byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 2)]; - for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) { - short length = (short) (rowOffset.get(rowId + 1) - rowOffset.get(rowId)); + byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 2)]; + for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { + short length = (short) (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); ByteUtil.setShort(data, offset, length); copyBytes(rowId, data, offset + 2, length); offset += 2 + length; @@ -423,8 +433,8 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { // output LV encoded byte array int offset = 0; byte[] data = new byte[totalLength]; - for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) { - int length = (rowOffset.get(rowId + 1) - rowOffset.get(rowId)); + for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) { + int length = (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId)); copyBytes(rowId, data, offset, length); offset += length; } @@ -452,4 +462,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage { capacity = newSize; } } + + /** + * free memory as needed + */ + public void freeMemory() { + if (null != rowOffset) { + rowOffset.freeMemory(); + rowOffset = null; + } + } }
