Immutable table SINGLE_CELL_ARRAY_WITH_OFFSETS values starting with separator byte return null in query results
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/edc9d12d Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/edc9d12d Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/edc9d12d Branch: refs/heads/system-catalog Commit: edc9d12dc056640526e388d7ccb1a6e2c6d3c51c Parents: 2759727 Author: Vincent Poon <vincentp...@apache.org> Authored: Wed Dec 27 15:34:32 2017 -0800 Committer: Vincent Poon <vincentp...@apache.org> Committed: Wed Dec 27 15:34:32 2017 -0800 ---------------------------------------------------------------------- .../apache/phoenix/end2end/StoreNullsIT.java | 6 +- .../phoenix/end2end/UpsertBigValuesIT.java | 64 ++++- .../phoenix/end2end/index/DropColumnIT.java | 19 +- .../phoenix/compile/ProjectionCompiler.java | 6 +- .../expression/SingleCellColumnExpression.java | 35 ++- .../apache/phoenix/index/IndexMaintainer.java | 2 +- .../NonAggregateRegionScannerFactory.java | 2 +- .../org/apache/phoenix/schema/ColumnRef.java | 4 +- .../java/org/apache/phoenix/schema/PTable.java | 21 +- .../phoenix/schema/types/PArrayDataType.java | 18 +- .../schema/types/PArrayDataTypeDecoder.java | 79 +++++- .../schema/types/PArrayDataTypeEncoder.java | 10 +- .../java/org/apache/phoenix/util/IndexUtil.java | 3 +- .../schema/ImmutableStorageSchemeTest.java | 241 +++++++++++++++++-- 14 files changed, 452 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java index 378a9ed..d1aee9f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java @@ -145,7 +145,11 @@ public class StoreNullsIT extends ParallelStatsDisabledIT { byte[] qualifier = table.getImmutableStorageScheme()== ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : nameColumn.getColumnQualifierBytes(); assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, qualifier)); assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column - KeyValueColumnExpression colExpression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? new SingleCellColumnExpression(nameColumn, "NAME", table.getEncodingScheme()) : new KeyValueColumnExpression(nameColumn); + KeyValueColumnExpression colExpression = + table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS + ? new SingleCellColumnExpression(nameColumn, "NAME", + table.getEncodingScheme(), table.getImmutableStorageScheme()) + : new KeyValueColumnExpression(nameColumn); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); colExpression.evaluate(new ResultTuple(rs), ptr); assertEquals(new ImmutableBytesPtr(PVarchar.INSTANCE.toBytes("v1")), ptr); http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java index ceb76d0..9842434 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java @@ -25,10 +25,16 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.util.Arrays; +import java.util.List; import java.util.Properties; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PSmallint; import org.junit.Test; +import com.google.common.collect.Lists; public class UpsertBigValuesIT extends ParallelStatsDisabledIT { @@ -374,4 +380,60 @@ public class UpsertBigValuesIT extends ParallelStatsDisabledIT { */ conn.close(); } -} + + @Test + public void testShort() throws Exception { + List<Short> testData = + Arrays.asList(Short.MIN_VALUE, Short.MAX_VALUE, (short) (Short.MIN_VALUE + 1), + (short) (Short.MAX_VALUE - 1), (short) 0, (short) 1, (short) -1); + testValues(false, PSmallint.INSTANCE, testData); + testValues(true, PSmallint.INSTANCE, testData); + } + + @Test + public void testBigInt() throws Exception { + List<Long> testData = + Arrays.asList(Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE + 1L, + Long.MAX_VALUE - 1L, 0L, 1L, -1L); + testValues(false, PLong.INSTANCE, testData); + testValues(true, PLong.INSTANCE, testData); + } + + private <T extends Number> void testValues(boolean immutable, PDataType<?> dataType, List<T> testData) throws Exception { + String tableName = generateUniqueName(); + String ddl = + String.format("CREATE %s TABLE %s (K INTEGER PRIMARY KEY, V1 %s)", + immutable ? "IMMUTABLE" : "", tableName, dataType.getSqlTypeName()); + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute(ddl); + String upsert = "UPSERT INTO " + tableName + " VALUES(?, ?)"; + PreparedStatement stmt = conn.prepareStatement(upsert); + int id = 1; + for (T testVal : testData) { + stmt.setInt(1, id++); + stmt.setObject(2, testVal, dataType.getSqlType()); + stmt.execute(); + } + conn.commit(); + String query = String.format("SELECT K,V1 FROM %s ORDER BY K ASC", tableName); + ResultSet rs = conn.createStatement().executeQuery(query); + int index = 0; + boolean failed = false; + List<String> errors = Lists.newArrayList(); + while (rs.next()) { + Number resultVal = rs.getObject(2, testData.get(0).getClass()); + T testVal = testData.get(index++); + if (!testVal.equals(resultVal)) { + errors.add(String.format("[expected=%s actual=%s] ", + testVal, resultVal)); + failed = true; + } + } + String errorMsg = + String.format("Data in table didn't match input: immutable=%s, dataType=%s, %s", + immutable, dataType.getSqlTypeName(), errors); + assertFalse(errorMsg, failed); + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java index badb2a6..28aa1e9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java @@ -254,7 +254,9 @@ public class DropColumnIT extends ParallelStatsDisabledIT { assertNotNull(result); byte[] colValue; if (!mutable && columnEncoded) { - KeyValueColumnExpression colExpression = new SingleCellColumnExpression(dataColumn, "V2", dataTable.getEncodingScheme()); + KeyValueColumnExpression colExpression = + new SingleCellColumnExpression(dataColumn, "V2", dataTable.getEncodingScheme(), + dataTable.getImmutableStorageScheme()); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); colExpression.evaluate(new ResultTuple(result), ptr); colValue = ptr.copyBytesIfNecessary(); @@ -273,7 +275,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT { result = results.next(); assertNotNull(result); if (!mutable && columnEncoded) { - KeyValueColumnExpression colExpression = new SingleCellColumnExpression(glovalIndexCol, "0:V2", globalIndexTable.getEncodingScheme()); + KeyValueColumnExpression colExpression = + new SingleCellColumnExpression(glovalIndexCol, "0:V2", + globalIndexTable.getEncodingScheme(), + globalIndexTable.getImmutableStorageScheme()); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); colExpression.evaluate(new ResultTuple(result), ptr); colValue = ptr.copyBytesIfNecessary(); @@ -293,7 +298,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT { result = results.next(); assertNotNull(result); if (!mutable && columnEncoded) { - KeyValueColumnExpression colExpression = new SingleCellColumnExpression(localIndexCol, "0:V2", localIndexTable.getEncodingScheme()); + KeyValueColumnExpression colExpression = + new SingleCellColumnExpression(localIndexCol, "0:V2", + localIndexTable.getEncodingScheme(), + localIndexTable.getImmutableStorageScheme()); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); assertTrue(colExpression.evaluate(new ResultTuple(result), ptr)); colValue = ptr.copyBytesIfNecessary(); @@ -387,7 +395,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT { PColumn localIndexCol = localIndex2.getColumnForColumnName(indexColumnName); byte[] colValue; if (!mutable && columnEncoded) { - KeyValueColumnExpression colExpression = new SingleCellColumnExpression(localIndexCol, indexColumnName, localIndex2.getEncodingScheme()); + KeyValueColumnExpression colExpression = + new SingleCellColumnExpression(localIndexCol, indexColumnName, + localIndex2.getEncodingScheme(), + localIndex2.getImmutableStorageScheme()); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); colExpression.evaluate(new ResultTuple(result), ptr); colValue = ptr.copyBytesIfNecessary(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index a147882..f85b5a8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -668,7 +668,11 @@ public class ProjectionCompiler { PTable table = context.getCurrentTable().getTable(); KeyValueColumnExpression keyValueColumnExpression; if (table.getImmutableStorageScheme() != ImmutableStorageScheme.ONE_CELL_PER_COLUMN) { - keyValueColumnExpression = new SingleCellColumnExpression(col, col.getName().getString(), table.getEncodingScheme()); + keyValueColumnExpression = + new SingleCellColumnExpression(col, + col.getName().getString(), + table.getEncodingScheme(), + table.getImmutableStorageScheme()); } else { keyValueColumnExpression = new KeyValueColumnExpression(col); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java index 8c1e0b6..e46e1fa 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java @@ -25,7 +25,6 @@ import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; import org.apache.phoenix.expression.visitor.ExpressionVisitor; @@ -54,12 +53,19 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { private String arrayColDisplayName; private KeyValueColumnExpression keyValueColumnExpression; private QualifierEncodingScheme encodingScheme; - + private ImmutableStorageScheme immutableStorageScheme; + public SingleCellColumnExpression() { } - - public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) { + + public SingleCellColumnExpression(ImmutableStorageScheme immutableStorageScheme) { + this.immutableStorageScheme = immutableStorageScheme; + } + + public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, + QualifierEncodingScheme encodingScheme, ImmutableStorageScheme immutableStorageScheme) { super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + this.immutableStorageScheme = immutableStorageScheme; Preconditions.checkNotNull(encodingScheme); Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); this.decodedColumnQualifier = encodingScheme.decode(cq); @@ -67,8 +73,9 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { setKeyValueExpression(); } - public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) { + public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme, ImmutableStorageScheme immutableStorageScheme) { super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + this.immutableStorageScheme = immutableStorageScheme; Preconditions.checkNotNull(encodingScheme); Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); this.arrayColDisplayName = displayName; @@ -86,8 +93,6 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { } // the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; - byte serializedImmutableStorageScheme = ptr.get()[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE]; - ImmutableStorageScheme immutableStorageScheme = ImmutableStorageScheme.fromSerializedValue(serializedImmutableStorageScheme); // Given a ptr to the entire array, set ptr to point to a particular element within that array ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); return encoderDecoder.decode(ptr, index); @@ -97,7 +102,18 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { public void readFields(DataInput input) throws IOException { super.readFields(input); this.decodedColumnQualifier = WritableUtils.readVInt(input); - this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)]; + int serializedEncodingScheme = WritableUtils.readVInt(input); + // prior to PHOENIX-4432 we weren't writing out the immutableStorageScheme in write(), + // so we use the decodedColumnQualifier sign to determine whether it's there + if (Integer.signum(serializedEncodingScheme) == -1) { + this.immutableStorageScheme = + ImmutableStorageScheme + .fromSerializedValue((byte) WritableUtils.readVInt(input)); + serializedEncodingScheme = -serializedEncodingScheme; + } else { + this.immutableStorageScheme = ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; + } + this.encodingScheme = QualifierEncodingScheme.values()[serializedEncodingScheme]; setKeyValueExpression(); } @@ -105,7 +121,8 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { public void write(DataOutput output) throws IOException { super.write(output); WritableUtils.writeVInt(output, decodedColumnQualifier); - WritableUtils.writeVInt(output, encodingScheme.ordinal()); + WritableUtils.writeVInt(output, -encodingScheme.ordinal()); //negative since PHOENIX-4432 + WritableUtils.writeVInt(output, immutableStorageScheme.getSerializedMetadataValue()); } public KeyValueColumnExpression getKeyValueExpression() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index 500ac4b..fa60679 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -1010,7 +1010,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { public PDataType getDataType() { return null; } - }, dataColRef.getFamily(), dataColRef.getQualifier(), encodingScheme); + }, dataColRef.getFamily(), dataColRef.getQualifier(), encodingScheme, immutableStorageScheme); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); expression.evaluate(new ValueGetterTuple(valueGetter, ts), ptr); byte[] value = ptr.copyBytesIfNecessary(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java index ded33cc..c097d0d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java @@ -209,7 +209,7 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory { int arrayKVRefSize = WritableUtils.readVInt(input); for (int i = 0; i < arrayKVRefSize; i++) { PTable.ImmutableStorageScheme scheme = EncodedColumnsUtil.getImmutableStorageScheme(scan); - KeyValueColumnExpression kvExp = scheme != PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN ? new SingleCellColumnExpression() + KeyValueColumnExpression kvExp = scheme != PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN ? new SingleCellColumnExpression(scheme) : new KeyValueColumnExpression(); kvExp.readFields(input); arrayKVRefs.add(kvExp); http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java index c73b860..fa3156d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java @@ -126,7 +126,9 @@ public class ColumnRef { } Expression expression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? - new SingleCellColumnExpression(column, displayName, table.getEncodingScheme()) : new KeyValueColumnExpression(column, displayName); + new SingleCellColumnExpression(column, displayName, + table.getEncodingScheme(), table.getImmutableStorageScheme()) + : new KeyValueColumnExpression(column, displayName); if (column.getExpressionStr() != null) { String url = PhoenixRuntime.JDBC_PROTOCOL http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index ec931b7..7e186ad 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -190,14 +190,14 @@ public interface PTable extends PMetaDataEntity { } }, // stores a single cell per column family that contains all serialized column values - SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2) { + SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2, PArrayDataType.IMMUTABLE_SERIALIZATION_V2) { @Override public ColumnValueEncoder getEncoder(int numElements) { PDataType type = PVarbinary.INSTANCE; int estimatedSize = PArrayDataType.estimateSize(numElements, type); TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize); DataOutputStream oStream = new DataOutputStream(byteStream); - return new PArrayDataTypeEncoder(byteStream, oStream, numElements, type, SortOrder.ASC, false, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION); + return new PArrayDataTypeEncoder(byteStream, oStream, numElements, type, SortOrder.ASC, false, getSerializationVersion()); } @Override @@ -207,15 +207,30 @@ public interface PTable extends PMetaDataEntity { }; private final byte serializedValue; - + private byte serializationVersion; + private ImmutableStorageScheme(byte serializedValue) { this.serializedValue = serializedValue; } + private ImmutableStorageScheme(byte serializedValue, byte serializationVersion) { + this.serializedValue = serializedValue; + this.serializationVersion = serializationVersion; + } + public byte getSerializedMetadataValue() { return this.serializedValue; } + public byte getSerializationVersion() { + return this.serializationVersion; + } + + @VisibleForTesting + void setSerializationVersion(byte serializationVersion) { + this.serializationVersion = serializationVersion; + } + public static ImmutableStorageScheme fromSerializedValue(byte serializedValue) { if (serializedValue < 1 || serializedValue > ImmutableStorageScheme.values().length) { return null; http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java index 7d742e2..162b235 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java @@ -75,7 +75,12 @@ public abstract class PArrayDataType<T> extends PDataType<T> { // array serialization format where bytes can be used as part of the row key public static final byte SORTABLE_SERIALIZATION_VERSION = 1; // array serialization format where bytes are immutable (does not support prepend/append or sorting) + @Deprecated public static final byte IMMUTABLE_SERIALIZATION_VERSION = 2; + // array serialization format where bytes are immutable (does not support prepend/append or sorting) + // differs from V1 in that nulls are not serialized + // we rely only on offsets to determine the presence of nulls + public static final byte IMMUTABLE_SERIALIZATION_V2 = 3; protected PArrayDataType(String sqlTypeName, int sqlType, Class clazz, PDataCodec codec, int ordinal) { super(sqlTypeName, sqlType, clazz, codec, ordinal); @@ -217,7 +222,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> { } public static boolean useShortForOffsetArray(int maxoffset, byte serializationVersion) { - if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION) { + if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION || serializationVersion == IMMUTABLE_SERIALIZATION_V2) { return (maxoffset <= Short.MAX_VALUE && maxoffset >= Short.MIN_VALUE ); } // If the max offset is less than Short.MAX_VALUE then offset array can use short @@ -383,7 +388,10 @@ public abstract class PArrayDataType<T> extends PDataType<T> { int offset; if (useShort) { offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex); - return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? 0 : Short.MAX_VALUE); + return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION + || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2 ? 0 + : Short.MAX_VALUE); } else { offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex); return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT); @@ -964,7 +972,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> { } } else { for (int pos : offsetPos) { - short val = serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? (short)pos : (short)(pos - Short.MAX_VALUE); + short val = + serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION + || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2 + ? (short) pos + : (short) (pos - Short.MAX_VALUE); Bytes.putShort(offsetArr, off, val); off += Bytes.SIZEOF_SHORT; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java index 7a6ea91..22fa46c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnValueDecoder; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.util.ByteUtil; @@ -78,13 +79,45 @@ public class PArrayDataTypeDecoder implements ColumnValueDecoder { } int elementLength = 0; if (arrayIndex == (noOfElements - 1)) { - int separatorBytes = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 3 : 0; - elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : indexOffset - - (currOffset + initPos) - separatorBytes; + // in the original IMMUTABLE_SERIALIZATION_VERSION (v1), for nulls we store + // (separatorByte, #_of_nulls) in the data. Because of the separatorByte, we can't + // distinguish between nulls and actual data values that start with the separator + // byte. We do a hack here to limit the damage by checking offsets - if the prior + // offset had a length of 0, then we know we're storing 2 or more nulls. However, we + // still can't fix the case distinguishing a single null from a short value. There + // are two kinds of separatorByte, so the results will be potentially incorrect for + // 2 short values that correspond to (separatorByte, 1) + if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) { + elementLength = indexOffset - (currOffset + initPos); + if (isNullValue(arrayIndex, bytes, initPos, serializationVersion, useShort, indexOffset, currOffset, elementLength)) { + elementLength = 0; + } + } else { + int separatorBytes = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 3 : 0; + elementLength = isSeparatorByte(bytes, initPos, currOffset) && serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 0 : indexOffset + - (currOffset + initPos) - separatorBytes; + } } else { - int separatorByte = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 1 : 0; - elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : PArrayDataType.getOffset(bytes, - arrayIndex + 1, useShort, indexOffset, serializationVersion) - currOffset - separatorByte; + if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) { + elementLength = PArrayDataType.getOffset(bytes, arrayIndex + 1, + useShort, indexOffset, serializationVersion) + - currOffset; + if (isNullValue(arrayIndex, bytes, initPos, serializationVersion, useShort, indexOffset, currOffset, elementLength)) { + elementLength = 0; + } + } else { + int separatorByte = + serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION + ? 1 + : 0; + elementLength = + isSeparatorByte(bytes, initPos, currOffset) + && serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION + ? 0 + : PArrayDataType.getOffset(bytes, arrayIndex + 1, + useShort, indexOffset, serializationVersion) + - currOffset - separatorByte; + } } ptr.set(bytes, currOffset + initPos, elementLength); } else { @@ -99,4 +132,38 @@ public class PArrayDataTypeDecoder implements ColumnValueDecoder { return true; } + // returns true if the prior element in the array is a null + private static boolean isNullValue(int arrayIndex, byte[] bytes, int initPos, + byte serializationVersion, boolean useShort, int indexOffset, int currOffset, + int elementLength) { + if (isSeparatorByte(bytes, initPos, currOffset)) { + if (isPriorValueZeroLength(arrayIndex, bytes, + serializationVersion, useShort, indexOffset, currOffset)) { + return true; + } else { + // if there's no prior null, there can be at most 1 null + if (elementLength == 2) { + // nullByte calculation comes from the encoding of one null + // see PArrayDataType#serializeNulls + byte nullByte = SortOrder.invert((byte)(0)); + if (bytes[initPos+currOffset+1] == nullByte) { + return true; + } + } + } + } + return false; + } + + // checks prior value length by subtracting offset of the previous item from the current offset + private static boolean isPriorValueZeroLength(int arrayIndex, byte[] bytes, byte serializationVersion, + boolean useShort, int indexOffset, int currOffset) { + return arrayIndex > 0 && currOffset - PArrayDataType.getOffset(bytes, arrayIndex - 1, + useShort, indexOffset, serializationVersion) == 0; + } + + private static boolean isSeparatorByte(byte[] bytes, int initPos, int currOffset) { + return bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE; + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java index 3dad6c3..7467981 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java @@ -92,7 +92,9 @@ public class PArrayDataTypeEncoder implements ColumnValueEncoder { // used to represent the absence of a value @Override public void appendAbsentValue() { - if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION && !baseType.isFixedWidth()) { + if ((serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION + || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2) + && !baseType.isFixedWidth()) { offsetPos.add(-byteStream.size()); nulls++; } @@ -125,7 +127,11 @@ public class PArrayDataTypeEncoder implements ColumnValueEncoder { offsetPos.add(byteStream.size()); nulls++; } else { - nulls = PArrayDataType.serializeNulls(oStream, nulls); + // we don't serialize nulls for IMMUTABLE_SERIALIZATION_V2 + if (serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION + || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) { + nulls = PArrayDataType.serializeNulls(oStream, nulls); + } offsetPos.add(byteStream.size()); if (sortOrder == SortOrder.DESC) { SortOrder.invert(bytes, offset, bytes, offset, len); http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 74f91b4..33b7383 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -468,13 +468,14 @@ public class IndexUtil { KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan); boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null; QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan); + ImmutableStorageScheme immutableStorageScheme = EncodedColumnsUtil.getImmutableStorageScheme(scan); Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length]; for (int i = 0; i < dataColumns.length; i++) { byte[] family = dataColumns[i].getFamily(); byte[] qualifier = dataColumns[i].getQualifier(); Field field = keyValueSchema.getField(i); Expression dataColumnExpr = - storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme) + storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme, immutableStorageScheme) : new KeyValueColumnExpression(field, family, qualifier); colExpressions[i] = dataColumnExpr; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/edc9d12d/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java index d8c5cdb..3d1b176 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java @@ -17,9 +17,15 @@ */ package org.apache.phoenix.schema; +import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; +import static org.apache.phoenix.schema.types.PArrayDataType.IMMUTABLE_SERIALIZATION_V2; +import static org.apache.phoenix.schema.types.PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION; +import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Arrays; import java.util.List; @@ -33,6 +39,11 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PSmallint; +import org.apache.phoenix.schema.types.PTinyint; +import org.apache.phoenix.schema.types.PUnsignedTinyint; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.ByteUtil; import org.junit.Test; @@ -55,16 +66,22 @@ public class ImmutableStorageSchemeTest { } }; private ImmutableStorageScheme immutableStorageScheme; + byte serializationVersion; - @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0}}") // name is used by failsafe as file name in reports - public static ImmutableStorageScheme[] data() { - ImmutableStorageScheme[] values = ImmutableStorageScheme.values(); - // skip ONE_CELL_PER_COLUMN - return Arrays.copyOfRange(values, 1, values.length); + @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0},serializationVersion={1}}") // name is used by failsafe as file name in reports + public static List<Object[]> data() { + return Arrays.asList(new Object[][] { + { SINGLE_CELL_ARRAY_WITH_OFFSETS, + IMMUTABLE_SERIALIZATION_VERSION }, + { SINGLE_CELL_ARRAY_WITH_OFFSETS, + IMMUTABLE_SERIALIZATION_V2 } + }); } - public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme) { + public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme, byte serializationVersion) { this.immutableStorageScheme = immutableStorageScheme; + this.immutableStorageScheme.setSerializationVersion(serializationVersion); + this.serializationVersion = serializationVersion; } @Test @@ -75,9 +92,7 @@ public class ImmutableStorageSchemeTest { children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); children.add(FALSE_EVAL_EXPRESSION); children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); - SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); - ImmutableBytesPtr ptr = new ImmutableBytesPtr(); - singleCellConstructorExpression.evaluate(null, ptr); + ImmutableBytesPtr ptr = evaluate(children); ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); @@ -160,23 +175,201 @@ public class ImmutableStorageSchemeTest { children.add(nullExpression); children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); - SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); + ImmutableBytesPtr ptr = evaluate(children); + + assertDecodedContents(ptr, new byte[][] {EMPTY_BYTE_ARRAY, EMPTY_BYTE_ARRAY, BYTE_ARRAY1, BYTE_ARRAY2}); + } + + @Test + public void testTrailingNulls() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); + children.add(nullExpression); + children.add(nullExpression); + ImmutableBytesPtr ptr = evaluate(children); + + assertDecodedContents(ptr, new byte[][] {BYTE_ARRAY1, BYTE_ARRAY2, EMPTY_BYTE_ARRAY, EMPTY_BYTE_ARRAY}); + } + + @Test + public void testManyNulls() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + byte[][] testData = new byte[300][]; + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + testData[0] = BYTE_ARRAY1; + for (int i = 1; i < testData.length - 1; i++) { + children.add(nullExpression); + testData[i] = EMPTY_BYTE_ARRAY; + } + children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); + testData[299] = BYTE_ARRAY2; + ImmutableBytesPtr ptr = evaluate(children); + + assertDecodedContents(ptr, testData); + } + + @Test + public void testSingleLeadingTrailingNull() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(nullExpression); + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + children.add(nullExpression); + ImmutableBytesPtr ptr = evaluate(children); + + assertDecodedContents(ptr, + new byte[][] { EMPTY_BYTE_ARRAY, BYTE_ARRAY1, EMPTY_BYTE_ARRAY }); + } + + @Test + public void testSingleMiddleNull() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE)); + ImmutableBytesPtr ptr = evaluate(children); + + assertDecodedContents(ptr, new byte[][] { BYTE_ARRAY1, EMPTY_BYTE_ARRAY, BYTE_ARRAY2 }); + } + + @Test + public void testAllShortValues() throws Exception { + int curr = Short.MIN_VALUE; + List<Expression> children = Lists.newArrayListWithExpectedSize(1); + List<Integer> failedValues = Lists.newArrayList(); + while (curr <= Short.MAX_VALUE) { + children.add(LiteralExpression.newConstant(curr, PSmallint.INSTANCE)); + ImmutableBytesPtr ptr = evaluate(children); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptr, 0)); + if (ptr.getLength() == 0) { + failedValues.add(curr); + } else { + if (curr != PSmallint.INSTANCE.getCodec().decodeShort(ptr.copyBytesIfNecessary(), 0, + SortOrder.ASC)) { + failedValues.add(curr); + } + } + children.remove(0); + curr++; + } + // in v1, we can't distinguish a null from two short values + if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION) { + assertTrue(failedValues.size() + " values were not properly decoded: " + failedValues, + failedValues.size() == 2); + } else { + assertTrue(failedValues.size() + " values were not properly decoded: " + failedValues, + failedValues.size() == 0); + } + } + + @Test + public void testSingleByteValues() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((byte) -128, PTinyint.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((byte) 0, PUnsignedTinyint.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((byte) 127, PUnsignedTinyint.INSTANCE)); + ImmutableBytesPtr ptr = evaluate(children); + + assertNullAtIndex(ptr, 0); + assertValueAtIndex(ptr, 1, (byte) -128, PTinyint.INSTANCE); + assertNullAtIndex(ptr, 2); + assertValueAtIndex(ptr, 3, (byte) 0, PUnsignedTinyint.INSTANCE); + assertNullAtIndex(ptr, 4); + assertValueAtIndex(ptr, 5, (byte) 127, PUnsignedTinyint.INSTANCE); + } + + @Test + public void testSeparatorByteValues() throws Exception { + List<Expression> children = Lists.newArrayListWithExpectedSize(4); + LiteralExpression nullExpression = LiteralExpression.newConstant(null); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((short) -32513, PSmallint.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((short) 32767, PSmallint.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant(Integer.MAX_VALUE, PInteger.INSTANCE)); + children.add(nullExpression); + children.add(LiteralExpression.newConstant(Integer.MIN_VALUE, PInteger.INSTANCE)); + // see if we can differentiate two nulls and {separatorByte, 2} + children.add(nullExpression); + children.add(nullExpression); + children.add(LiteralExpression.newConstant((short) -32514, PSmallint.INSTANCE)); + + ImmutableBytesPtr ptr = evaluate(children); + + assertNullAtIndex(ptr, 0); + try { + assertValueAtIndex(ptr, 1, (short) -32513, PSmallint.INSTANCE); + } catch (Exception e) { + if (serializationVersion != IMMUTABLE_SERIALIZATION_VERSION) { + fail("Failed on exception " + e); + } + } + assertNullAtIndex(ptr, 2); + try { + assertValueAtIndex(ptr, 3, (short) 32767, PSmallint.INSTANCE); + } catch (Exception e) { + if (serializationVersion != IMMUTABLE_SERIALIZATION_VERSION) { + fail("Failed on exception " + e); + } + } + assertNullAtIndex(ptr, 4); + assertValueAtIndex(ptr, 5, Integer.MAX_VALUE, PInteger.INSTANCE); + assertNullAtIndex(ptr, 6); + assertValueAtIndex(ptr, 7, Integer.MIN_VALUE, PInteger.INSTANCE); + assertNullAtIndex(ptr, 8); + assertNullAtIndex(ptr, 9); + assertValueAtIndex(ptr, 10, (short) -32514, PSmallint.INSTANCE); + } + + private void assertNullAtIndex(ImmutableBytesPtr ptr, int index) { + assertValueAtIndex(ptr, index, null, null); + } + + private void assertValueAtIndex(ImmutableBytesPtr ptr, int index, Object value, + PDataType type) { + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); + assertTrue(decoder.decode(ptrCopy, index)); + if (value == null) { + assertArrayEquals(EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); + return; + } + Object decoded; + if (type.equals(PSmallint.INSTANCE)) { + decoded = type.getCodec().decodeShort(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC); + } else if (type.equals(PInteger.INSTANCE)) { + decoded = type.getCodec().decodeInt(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC); + } else { // assume byte for all other types + decoded = type.getCodec().decodeByte(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC); + } + assertEquals(value, decoded); + } + + private ImmutableBytesPtr evaluate(List<Expression> children) { + SingleCellConstructorExpression singleCellConstructorExpression = + new SingleCellConstructorExpression(immutableStorageScheme, children); ImmutableBytesPtr ptr = new ImmutableBytesPtr(); singleCellConstructorExpression.evaluate(null, ptr); - - ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + return ptr; + } + + private void assertDecodedContents(ImmutableBytesPtr ptr, byte[]... contents) { ColumnValueDecoder decoder = immutableStorageScheme.getDecoder(); - assertTrue(decoder.decode(ptrCopy, 0)); - assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); - ptrCopy = new ImmutableBytesPtr(ptr); - assertTrue(decoder.decode(ptrCopy, 1)); - assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary()); - ptrCopy = new ImmutableBytesPtr(ptr); - assertTrue(decoder.decode(ptrCopy, 2)); - assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary()); - ptrCopy = new ImmutableBytesPtr(ptr); - assertTrue(decoder.decode(ptrCopy, 3)); - assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary()); + for (int i = 0; i < contents.length; i++) { + ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr); + assertTrue(decoder.decode(ptrCopy, i)); + assertArrayEquals(contents[i], ptrCopy.copyBytesIfNecessary()); + } } - + }
