Repository: phoenix Updated Branches: refs/heads/encodecolumns2 e89337f83 -> a65ab0030
PHOENIX-3666 Make use of EncodedColumnQualifierCellsList for all column name mapping schemes Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a65ab003 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a65ab003 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a65ab003 Branch: refs/heads/encodecolumns2 Commit: a65ab0030eb50315d8e82948bcf334fc41cea575 Parents: e89337f Author: Samarth <samarth.j...@salesforce.com> Authored: Mon Feb 13 15:51:10 2017 -0800 Committer: Samarth <samarth.j...@salesforce.com> Committed: Mon Feb 13 15:51:10 2017 -0800 ---------------------------------------------------------------------- .../apache/phoenix/query/QueryConstants.java | 22 +++---- .../java/org/apache/phoenix/schema/PName.java | 26 -------- .../java/org/apache/phoenix/schema/PTable.java | 62 +++++++++++++++++++- .../apache/phoenix/util/EncodedColumnsUtil.java | 9 ++- .../util/QualifierEncodingSchemeTest.java | 10 ---- 5 files changed, 77 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a65ab003/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 9f4a569..6f105f1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -48,6 +48,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_CO import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; @@ -88,7 +89,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; @@ -108,7 +108,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE; -import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS; import java.math.BigDecimal; @@ -122,6 +121,7 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.schema.MetaDataSplitPolicy; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.SortOrder; @@ -161,23 +161,19 @@ public interface QueryConstants { /** BEGIN Set of reserved column qualifiers **/ - public static final String RESERVED_COLUMN_FAMILY = "_r"; + public static final String RESERVED_COLUMN_FAMILY = "_v"; public static final byte[] RESERVED_COLUMN_FAMILY_BYTES = Bytes.toBytes(RESERVED_COLUMN_FAMILY); public static final byte[] VALUE_COLUMN_FAMILY = RESERVED_COLUMN_FAMILY_BYTES; - //TODO: samarth think about the implication of using the four byte scheme here. Can we just - // get away with storing them in a single byte? We would need to make our encoding scheme - // cognizant of the fact that all bytes may not be available making them interoperable. - // In other words allow upper casting but not downcasting. - public static final byte[] VALUE_COLUMN_QUALIFIER = FOUR_BYTE_QUALIFIERS.encode(1); + public static final byte[] VALUE_COLUMN_QUALIFIER = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(1); public static final byte[] ARRAY_VALUE_COLUMN_FAMILY = RESERVED_COLUMN_FAMILY_BYTES; - public static final byte[] ARRAY_VALUE_COLUMN_QUALIFIER = FOUR_BYTE_QUALIFIERS.encode(2); + public static final byte[] ARRAY_VALUE_COLUMN_QUALIFIER = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(2); public final static PName SINGLE_COLUMN_NAME = PNameFactory.newNormalizedName("s"); public final static PName SINGLE_COLUMN_FAMILY_NAME = PNameFactory.newNormalizedName("s"); - public final static byte[] SINGLE_COLUMN = FOUR_BYTE_QUALIFIERS.encode(3); - public final static byte[] SINGLE_COLUMN_FAMILY = RESERVED_COLUMN_FAMILY_BYTES; + public final static byte[] SINGLE_COLUMN = SINGLE_COLUMN_NAME.getBytes(); + public final static byte[] SINGLE_COLUMN_FAMILY = SINGLE_COLUMN_FAMILY_NAME.getBytes(); /** END Set of reserved column qualifiers **/ @@ -208,9 +204,7 @@ public interface QueryConstants { public static final ImmutableBytesPtr EMPTY_COLUMN_BYTES_PTR = new ImmutableBytesPtr( EMPTY_COLUMN_BYTES); public static final Integer ENCODED_EMPTY_COLUMN_NAME = 0; - public static final byte[] ENCODED_EMPTY_COLUMN_BYTES = FOUR_BYTE_QUALIFIERS.encode(ENCODED_EMPTY_COLUMN_NAME); - public static final ImmutableBytesPtr ENCODED_EMPTY_COLUMN_BYTES_PTR = new ImmutableBytesPtr( - ENCODED_EMPTY_COLUMN_BYTES); + public static final byte[] ENCODED_EMPTY_COLUMN_BYTES = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.encode(ENCODED_EMPTY_COLUMN_NAME); public final static String EMPTY_COLUMN_VALUE = "x"; public final static byte[] EMPTY_COLUMN_VALUE_BYTES = Bytes.toBytes(EMPTY_COLUMN_VALUE); public static final ImmutableBytesPtr EMPTY_COLUMN_VALUE_BYTES_PTR = new ImmutableBytesPtr( http://git-wip-us.apache.org/repos/asf/phoenix/blob/a65ab003/phoenix-core/src/main/java/org/apache/phoenix/schema/PName.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PName.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PName.java index 8df6a95..0e1337c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PName.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PName.java @@ -83,32 +83,6 @@ public interface PName { return 0; } }; - public static PName ENCODED_EMPTY_COLUMN_NAME = new PName() { - @Override - public String getString() { - return String.valueOf(QueryConstants.ENCODED_EMPTY_COLUMN_NAME); - } - - @Override - public byte[] getBytes() { - return QueryConstants.ENCODED_EMPTY_COLUMN_BYTES; - } - - @Override - public String toString() { - return getString(); - } - - @Override - public ImmutableBytesPtr getBytesPtr() { - return QueryConstants.ENCODED_EMPTY_COLUMN_BYTES_PTR; - } - - @Override - public int getEstimatedSize() { - return 0; - } - }; /** * Get the client-side, normalized name as referenced * in a SQL statement. http://git-wip-us.apache.org/repos/asf/phoenix/blob/a65ab003/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index 4a02e54..d8badf8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -17,7 +17,9 @@ */ package org.apache.phoenix.schema; +import static com.google.common.base.Preconditions.checkArgument; import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; +import static org.apache.phoenix.util.EncodedColumnsUtil.isReservedColumnQualifier; import java.io.DataOutputStream; import java.util.Collections; @@ -251,6 +253,9 @@ public interface PTable extends PMetaDataEntity { @Override public byte[] encode(int value) { + if (isReservedColumnQualifier(value)) { + return FOUR_BYTE_QUALIFIERS.encode(value); + } if (value < 0 || value > maxQualifier) { throw new QualifierOutOfRangeException(0, maxQualifier); } @@ -259,6 +264,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes) { + if (bytes.length == 4) { + return getReservedQualifier(bytes); + } if (bytes.length != 1) { throw new InvalidQualifierBytesException(1, bytes.length); } @@ -267,6 +275,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes, int offset, int length) { + if (length == 4) { + return getReservedQualifier(bytes, offset, length); + } if (length != 1) { throw new InvalidQualifierBytesException(1, length); } @@ -283,6 +294,9 @@ public interface PTable extends PMetaDataEntity { @Override public byte[] encode(int value) { + if (isReservedColumnQualifier(value)) { + return FOUR_BYTE_QUALIFIERS.encode(value); + } if (value < 0 || value > maxQualifier) { throw new QualifierOutOfRangeException(0, maxQualifier); } @@ -291,6 +305,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes) { + if (bytes.length == 4) { + return getReservedQualifier(bytes); + } if (bytes.length != 2) { throw new InvalidQualifierBytesException(2, bytes.length); } @@ -299,6 +316,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes, int offset, int length) { + if (length == 4) { + return getReservedQualifier(bytes, offset, length); + } if (length != 2) { throw new InvalidQualifierBytesException(2, length); } @@ -313,6 +333,9 @@ public interface PTable extends PMetaDataEntity { THREE_BYTE_QUALIFIERS((byte)3, 16777215) { @Override public byte[] encode(int value) { + if (isReservedColumnQualifier(value)) { + return FOUR_BYTE_QUALIFIERS.encode(value); + } if (value < 0 || value > maxQualifier) { throw new QualifierOutOfRangeException(0, maxQualifier); } @@ -322,6 +345,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes) { + if (bytes.length == 4) { + return getReservedQualifier(bytes); + } if (bytes.length != 3) { throw new InvalidQualifierBytesException(2, bytes.length); } @@ -334,6 +360,9 @@ public interface PTable extends PMetaDataEntity { @Override public int decode(byte[] bytes, int offset, int length) { + if (length == 4) { + return getReservedQualifier(bytes, offset, length); + } if (length != 3) { throw new InvalidQualifierBytesException(3, length); } @@ -417,6 +446,36 @@ public interface PTable extends PMetaDataEntity { super("Invalid number of qualifier bytes. Expected length: " + expectedLength + ". Actual: " + actualLength); } } + + /** + * We generate our column qualifiers in the reserved range 0-10 using the FOUR_BYTE_QUALIFIERS + * encoding. When adding Cells corresponding to the reserved qualifiers to the + * EncodedColumnQualifierCells list, we need to make sure that we use the FOUR_BYTE_QUALIFIERS + * scheme to decode the correct int value. + */ + private static int getReservedQualifier(byte[] bytes) { + checkArgument(bytes.length == 4); + int number = FOUR_BYTE_QUALIFIERS.decode(bytes); + if (!isReservedColumnQualifier(number)) { + throw new InvalidQualifierBytesException(4, bytes.length); + } + return number; + } + + /** + * We generate our column qualifiers in the reserved range 0-10 using the FOUR_BYTE_QUALIFIERS + * encoding. When adding Cells corresponding to the reserved qualifiers to the + * EncodedColumnQualifierCells list, we need to make sure that we use the FOUR_BYTE_QUALIFIERS + * scheme to decode the correct int value. + */ + private static int getReservedQualifier(byte[] bytes, int offset, int length) { + checkArgument(length == 4); + int number = FOUR_BYTE_QUALIFIERS.decode(bytes, offset, length); + if (!isReservedColumnQualifier(number)) { + throw new InvalidQualifierBytesException(4, length); + } + return number; + } } interface QualifierEncoderDecoder { @@ -425,7 +484,7 @@ public interface PTable extends PMetaDataEntity { int decode(byte[] bytes, int offset, int length); Integer getMaxQualifier(); } - + long getTimeStamp(); long getSequenceNumber(); long getIndexDisableTimestamp(); @@ -728,4 +787,5 @@ public interface PTable extends PMetaDataEntity { } } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a65ab003/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java index 725161a..b33b085 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java @@ -176,7 +176,7 @@ public class EncodedColumnsUtil { public static Expression[] createColumnExpressionArray(int maxEncodedColumnQualifier) { // reserve the first position and offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) - int numElements = maxEncodedColumnQualifier - QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+2; + int numElements = maxEncodedColumnQualifier - QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE + 2; Expression[] colValues = new Expression[numElements]; Arrays.fill(colValues, new DelegateExpression(LiteralExpression.newConstant(null)) { @Override @@ -188,4 +188,11 @@ public class EncodedColumnsUtil { colValues[0]=LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES); return colValues; } + + public static boolean isReservedColumnQualifier(int number) { + if (number < 0) { + throw new IllegalArgumentException("Negative column qualifier" + number + " not allowed "); + } + return number < QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a65ab003/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java index 3dbc676..2b08d7d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java @@ -68,11 +68,6 @@ public class QualifierEncodingSchemeTest { TWO_BYTE_QUALIFIERS.decode(arr2); fail(); } catch (InvalidQualifierBytesException expected) {} - try { - TWO_BYTE_QUALIFIERS.decode(arr2, 0, 1); - fail(); - } catch (InvalidQualifierBytesException expected) {} - } @Test @@ -91,14 +86,9 @@ public class QualifierEncodingSchemeTest { byte[] arr2 = new byte[] {-128, arr1[0], arr1[1], arr1[2]}; assertEquals(16777215, THREE_BYTE_QUALIFIERS.decode(arr2, 1, 3)); try { - THREE_BYTE_QUALIFIERS.decode(arr2); - fail(); - } catch (InvalidQualifierBytesException expected) {} - try { THREE_BYTE_QUALIFIERS.decode(arr2, 0, 2); fail(); } catch (InvalidQualifierBytesException expected) {} - } @Test
