Repository: phoenix Updated Branches: refs/heads/4.x-cdh5.13 e63a313b1 -> 11a9b2472 (forced update)
PHOENIX-4737 Use position as column qualifier for APPEND_ONLY_SCHEMA Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a668715b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a668715b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a668715b Branch: refs/heads/4.x-cdh5.13 Commit: a668715b67c8c3dae09d7cdb03b6cce2c23daa7b Parents: 84106f4 Author: James Taylor <jtay...@salesforce.com> Authored: Sun May 13 17:36:21 2018 -0700 Committer: James Taylor <jtay...@salesforce.com> Committed: Tue May 15 12:37:30 2018 -0700 ---------------------------------------------------------------------- .../end2end/ColumnEncodedBytesPropIT.java | 60 ++++++++++++++++++++ .../coprocessor/MetaDataEndpointImpl.java | 9 ++- .../apache/phoenix/schema/MetaDataClient.java | 11 ++-- 3 files changed, 75 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a668715b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedBytesPropIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedBytesPropIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedBytesPropIT.java index 3b129f5..c85ff6e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedBytesPropIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedBytesPropIT.java @@ -17,22 +17,29 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Test; @@ -109,4 +116,57 @@ public class ColumnEncodedBytesPropIT extends ParallelStatsDisabledIT { } } + @Test + public void testAppendOnlySchema() throws SQLException { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + String dataTableFullName = SchemaUtil.getTableName("", generateUniqueName()); + String view1 = SchemaUtil.getTableName("", generateUniqueName()); + String view2 = SchemaUtil.getTableName("", generateUniqueName()); + try (Connection conn = DriverManager.getConnection(getUrl(), props);) { + Statement stmt = conn.createStatement(); + stmt.execute("CREATE IMMUTABLE TABLE " + dataTableFullName + + " (id varchar not null, v1 varchar " + + " CONSTRAINT pk PRIMARY KEY (id)) COLUMN_ENCODED_BYTES=2, APPEND_ONLY_SCHEMA=true, UPDATE_CACHE_FREQUENCY=NEVER"); + stmt.execute("ALTER TABLE " + dataTableFullName + " ADD v2 varchar"); + + stmt.execute("CREATE VIEW " + view1 + "(v3 varchar, v4 varchar)" + + " AS SELECT * FROM " + dataTableFullName + " WHERE v1='a'"); + stmt.execute("CREATE VIEW " + view2 + "(v3 bigint, v4 integer)" + + " AS SELECT * FROM " + dataTableFullName + " WHERE v1='b'"); + PTable v1 = PhoenixRuntime.getTable(conn, view1); + PTable v2 = PhoenixRuntime.getTable(conn, view1); + assertEquals(v1.getColumns().size(), v2.getColumns().size()); + for (int i = 1; i < v1.getColumns().size(); i++) { + PColumn c1 = v1.getColumns().get(i); + PColumn c2 = v2.getColumns().get(i); + assertEquals(ENCODED_CQ_COUNTER_INITIAL_VALUE + i - Math.abs(Short.MIN_VALUE), Bytes.toShort(c1.getColumnQualifierBytes())); + assertEquals(ENCODED_CQ_COUNTER_INITIAL_VALUE + i - Math.abs(Short.MIN_VALUE), Bytes.toShort(c2.getColumnQualifierBytes())); + } + + // add one more column to confirm disallowed now + try { + stmt.execute("ALTER TABLE " + dataTableFullName + " ADD v5 varchar"); + fail(); + } + catch (SQLException e) { + assertEquals(SQLExceptionCode.CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); + } + + conn.setAutoCommit(true); + stmt.execute("UPSERT INTO " + view1 + " VALUES('a','a','c','d','e')"); + stmt.execute("UPSERT INTO " + view2 + " VALUES('b','b','c',1, 2)"); + ResultSet rs1 = stmt.executeQuery("SELECT * FROM " + view1); + assertTrue(rs1.next()); + assertEquals("a",rs1.getString(1)); + assertEquals("d",rs1.getString(4)); + assertEquals("e",rs1.getString(5)); + assertFalse(rs1.next()); + ResultSet rs2 = stmt.executeQuery("SELECT * FROM " + view2); + assertTrue(rs2.next()); + assertEquals("b",rs2.getString(1)); + assertEquals(1L,rs2.getLong(4)); + assertEquals(2,rs2.getInt(5)); + assertFalse(rs2.next()); + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a668715b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index b77f113..ff62c92 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -2564,7 +2564,14 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso + Bytes.toString(schema) + ", table:" + Bytes.toString(table)); continue; - } + } + /* + * Disallow adding columns to a base table with APPEND_ONLY_SCHEMA since this + * creates a gap in the column positions for every view (PHOENIX-4737). + */ + if (!columnPutsForBaseTable.isEmpty() && view.isAppendOnlySchema()) { + return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), basePhysicalTable); + } ColumnOrdinalPositionUpdateList ordinalPositionList = new ColumnOrdinalPositionUpdateList(); List<PColumn> viewPkCols = new ArrayList<>(view.getPKColumns()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a668715b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 009289b..efbb939 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -92,6 +92,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_UPDATE_STATS_ASYNC; @@ -2367,7 +2368,9 @@ public class MetaDataClient { cqCounterFamily = defaultFamilyName != null ? defaultFamilyName : DEFAULT_COLUMN_FAMILY; } } - Integer encodedCQ = isPkColumn ? null : cqCounter.getNextQualifier(cqCounterFamily); + // Use position as column qualifier if APPEND_ONLY_SCHEMA to prevent gaps in + // the column encoding (PHOENIX-4737). + Integer encodedCQ = isPkColumn ? null : isAppendOnlySchema ? Integer.valueOf(ENCODED_CQ_COUNTER_INITIAL_VALUE + position) : cqCounter.getNextQualifier(cqCounterFamily); byte[] columnQualifierBytes = null; try { columnQualifierBytes = EncodedColumnsUtil.getColumnQualifierBytes(columnDefName.getColumnName(), encodedCQ, encodingScheme, isPkColumn); @@ -2378,7 +2381,7 @@ public class MetaDataClient { .setTableName(tableName).build().buildException(); } PColumn column = newColumn(position++, colDef, pkConstraint, defaultFamilyName, false, columnQualifierBytes, isImmutableRows); - if (cqCounter.increment(cqCounterFamily)) { + if (!isAppendOnlySchema && cqCounter.increment(cqCounterFamily)) { changedCqCounters.put(cqCounterFamily, cqCounter.getNextQualifier(cqCounterFamily)); } if (SchemaUtil.isPKColumn(column)) { @@ -3333,8 +3336,8 @@ public class MetaDataClient { } else { familyName = defaultColumnFamily; } - encodedCQ = cqCounterToUse.getNextQualifier(familyName); - if (cqCounterToUse.increment(familyName)) { + encodedCQ = table.isAppendOnlySchema() ? Integer.valueOf(ENCODED_CQ_COUNTER_INITIAL_VALUE + position) : cqCounterToUse.getNextQualifier(familyName); + if (!table.isAppendOnlySchema() && cqCounterToUse.increment(familyName)) { changedCqCounters.put(familyName, cqCounterToUse.getNextQualifier(familyName)); }
