This is an automated email from the ASF dual-hosted git repository. gokcen pushed a commit to branch 4.x-PHOENIX-5923 in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-PHOENIX-5923 by this push: new a2cd0a1 PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES a2cd0a1 is described below commit a2cd0a13ae5b11a13306e7f6d023559bc3f3e49c Author: Gokcen Iskender <gisken...@salesforce.com> AuthorDate: Wed Nov 11 16:37:39 2020 -0800 PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES Signed-off-by: Gokcen Iskender <gisken...@salesforce.com> --- .../org/apache/phoenix/end2end/CreateTableIT.java | 91 ++++++++++++++++++ .../org/apache/phoenix/schema/MetaDataClient.java | 104 +++++++++++++++------ 2 files changed, 167 insertions(+), 28 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java index 4f6ccab..c92be7c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java @@ -622,6 +622,97 @@ public class CreateTableIT extends ParallelStatsDisabledIT { } } + @Test + public void testCreateIndexWithDifferentStorageAndEncoding() throws Exception { + verifyIndexSchemeChange(false, false); + verifyIndexSchemeChange(false, true); + verifyIndexSchemeChange(true, false); + verifyIndexSchemeChange(true, true); + + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + String createTableDDL = "create IMMUTABLE TABLE " + tableName + "(id char(1) NOT NULL, col1 char(1), col2 char(1) " + + "CONSTRAINT NAME_PK PRIMARY KEY (id)) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; + String createIndexDDL = "create INDEX " + indexName + " ON " + tableName + " (col1) INCLUDE (col2) IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN"; + + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute(createTableDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, tableName, conn); + + boolean failed = false; + try { + conn.createStatement().execute(createIndexDDL); + } catch (SQLException e) { + assertEquals(e.getErrorCode(), SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE.getErrorCode()); + failed = true; + } + assertEquals(true, failed); + } + } + + private void verifyIndexSchemeChange(boolean immutable, boolean multiTenant) throws Exception{ + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + String nonEncodedOneCellPerColumnTable = generateUniqueName(); + String createTableDDL; + String createIndexDDL; + String tableName= "[TABLE_NAME]"; + String indexName= "[INDEX_NAME]"; + String createTableBaseDDL = "create " + (immutable? " IMMUTABLE ":"") + " TABLE [TABLE_NAME] (" + + " id char(1) NOT NULL," + " col1 integer NOT NULL," + + " col2 bigint NOT NULL," + + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)) MULTI_TENANT=" + (multiTenant? "true,":"false,"); + + String createIndexBaseDDL = "create index [INDEX_NAME] ON [TABLE_NAME] (col1) INCLUDE (col2) "; + + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + createTableDDL = createTableBaseDDL.replace(tableName, nonEncodedOneCellPerColumnTable); + createTableDDL += "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0"; + conn.createStatement().execute(createTableDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, + ImmutableStorageScheme.ONE_CELL_PER_COLUMN, + nonEncodedOneCellPerColumnTable, conn); + + String idxName = "IDX_" + generateUniqueName(); + // Don't specify anything to see if it inherits from parent + createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable); + conn.createStatement().execute(createIndexDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, + ImmutableStorageScheme.ONE_CELL_PER_COLUMN, + idxName, conn); + + idxName = "IDX_" + generateUniqueName(); + createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS"; + conn.createStatement().execute(createIndexDDL); + // Check if it sets the encoding to 2 + assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, + idxName, conn); + + idxName = "IDX_" + generateUniqueName(); + createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=3"; + conn.createStatement().execute(createIndexDDL); + assertColumnEncodingMetadata(QualifierEncodingScheme.THREE_BYTE_QUALIFIERS, + ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, + idxName, conn); + + createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable); + createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=0"; + // should fail + boolean failed = false; + try { + conn.createStatement().execute(createIndexDDL); + } catch (SQLException e) { + failed = true; + assertEquals(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES.getErrorCode(),e.getErrorCode()); + } + assertEquals(true, failed); + } + } + private void verifyUCFValueInSysCat(String tableName, String createTableString, Properties props, long expectedUCFInSysCat) throws SQLException { String readSysCatQuery = "SELECT TABLE_NAME, UPDATE_CACHE_FREQUENCY FROM SYSTEM.CATALOG " diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 0a45981..86a5cc1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -2534,28 +2534,46 @@ public class MetaDataClient { * */ if (parent != null) { - encodingScheme = parent.getEncodingScheme(); - immutableStorageScheme = parent.getImmutableStorageScheme(); - } else { - Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); - if (encodingSchemeSerializedByte == null) { - // Ignore default if transactional and column encoding is not supported (as with OMID) - if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, - QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); - encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); - } + Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); + // Table has encoding scheme defined + if (encodingSchemeSerializedByte != null) { + encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider); } else { - encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); - if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER) - .setSchemaName(schemaName).setTableName(tableName) - .setMessage(transactionProvider.name()) - .build() - .buildException(); + encodingScheme = parent.getEncodingScheme(); + } + + ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME.getValue(tableProps); + if (immutableStorageSchemeProp == null) { + immutableStorageScheme = parent.getImmutableStorageScheme(); + } else { + immutableStorageScheme = getImmutableStorageScheme(immutableStorageSchemeProp, schemaName, tableName, transactionProvider); + } + + if (immutableStorageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) { + if (encodingScheme == NON_ENCODED_QUALIFIERS) { + if (encodingSchemeSerializedByte != null) { + // encoding scheme is set as non-encoded on purpose, so we should fail + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES) + .setSchemaName(schemaName).setTableName(tableName).build().buildException(); + } else { + // encoding scheme is inherited from parent but it is not compatible with Single Cell. + encodingScheme = + QualifierEncodingScheme.fromSerializedValue( + (byte) QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); + } } } + + if (parent.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS && immutableStorageScheme == ONE_CELL_PER_COLUMN) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE) + .setSchemaName(schemaName).setTableName(tableName).build() + .buildException(); + } + LOGGER.info(String.format("STORAGE--ENCODING: %s--%s", immutableStorageScheme, encodingScheme)); + } else { + encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider); + if (isImmutableRows) { ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME @@ -2586,15 +2604,7 @@ public class MetaDataClient { } } } else { - immutableStorageScheme = immutableStorageSchemeProp; - if (immutableStorageScheme != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER) - .setSchemaName(schemaName).setTableName(tableName) - .setMessage(transactionProvider.name()) - .build() - .buildException(); - } + immutableStorageScheme = getImmutableStorageScheme(immutableStorageSchemeProp, schemaName, tableName, transactionProvider); } if (immutableStorageScheme != ONE_CELL_PER_COLUMN && encodingScheme == NON_ENCODED_QUALIFIERS) { @@ -3133,6 +3143,44 @@ public class MetaDataClient { } } + private QualifierEncodingScheme getEncodingScheme(Map<String, Object> tableProps, String schemaName, String tableName, TransactionFactory.Provider transactionProvider) + throws SQLException { + QualifierEncodingScheme encodingScheme = null; + Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps); + if (encodingSchemeSerializedByte == null) { + // Ignore default if transactional and column encoding is not supported (as with OMID) + if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { + encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB, + QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES); + encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); + } else { + encodingScheme = NON_ENCODED_QUALIFIERS; + } + } else { + encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte); + if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider() + .isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER) + .setSchemaName(schemaName).setTableName(tableName).setMessage(transactionProvider.name()).build().buildException(); + } + } + + return encodingScheme; + } + + private ImmutableStorageScheme getImmutableStorageScheme(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider) + throws SQLException { + if (immutableStorageSchemeProp != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER) + .setSchemaName(schemaName).setTableName(tableName) + .setMessage(transactionProvider.name()) + .build() + .buildException(); + } + return immutableStorageSchemeProp; + } + /* This method handles mutation codes sent by phoenix server, except for TABLE_NOT_FOUND which * is considered to be a success code. If TABLE_ALREADY_EXISTS in hbase, we don't need to add * it in ConnectionQueryServices and we return result as true. However if code is