Fix test failures
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c1958d07 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c1958d07 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c1958d07 Branch: refs/heads/encodecolumns2 Commit: c1958d07e4b8777d5ab7ff30f36450d19dce69a7 Parents: 23a8798 Author: Samarth <samarth.j...@salesforce.com> Authored: Wed Oct 19 11:57:48 2016 -0700 Committer: Samarth <samarth.j...@salesforce.com> Committed: Wed Oct 19 11:57:48 2016 -0700 ---------------------------------------------------------------------- .../AlterMultiTenantTableWithViewsIT.java | 25 +- .../apache/phoenix/end2end/AlterTableIT.java | 98 +++++--- .../phoenix/end2end/AlterTableWithViewsIT.java | 139 ++++++----- .../phoenix/end2end/StatsCollectorIT.java | 8 +- .../apache/phoenix/end2end/StoreNullsIT.java | 41 +++- .../phoenix/end2end/index/DropMetadataIT.java | 2 +- .../end2end/index/IndexExpressionIT.java | 7 +- .../end2end/index/MutableIndexFailureIT.java | 2 + .../phoenix/compile/CreateTableCompiler.java | 13 +- .../phoenix/compile/ExpressionCompiler.java | 4 +- .../compile/PostLocalIndexDDLCompiler.java | 7 +- .../apache/phoenix/compile/WhereCompiler.java | 21 -- .../GroupedAggregateRegionObserver.java | 14 +- .../coprocessor/MetaDataEndpointImpl.java | 95 +++++--- .../phoenix/coprocessor/ScanRegionObserver.java | 10 +- .../UngroupedAggregateRegionObserver.java | 4 +- .../expression/ArrayColumnExpression.java | 116 +++++---- .../expression/KeyValueColumnExpression.java | 2 +- ...lumnWithKeyValueColumnExpressionVisitor.java | 37 --- .../filter/MultiKeyValueComparisonFilter.java | 5 - .../filter/SingleKeyValueComparisonFilter.java | 6 - .../apache/phoenix/index/IndexMaintainer.java | 108 +++++---- .../apache/phoenix/index/PhoenixIndexCodec.java | 2 +- .../index/PhoenixIndexFailurePolicy.java | 5 +- .../phoenix/iterate/BaseResultIterators.java | 1 - .../iterate/RegionScannerResultIterator.java | 4 +- .../apache/phoenix/schema/MetaDataClient.java | 127 +++++----- .../tuple/EncodedColumnQualiferCellsList.java | 71 ++++-- .../tuple/PositionBasedMultiKeyValueTuple.java | 4 +- .../schema/tuple/PositionBasedResultTuple.java | 1 + .../apache/phoenix/util/EncodedColumnsUtil.java | 7 +- .../java/org/apache/phoenix/util/IndexUtil.java | 240 ++++++++++--------- .../org/apache/phoenix/util/MetaDataUtil.java | 6 + .../java/org/apache/phoenix/util/ScanUtil.java | 4 +- .../phoenix/index/IndexMaintainerTest.java | 2 +- 35 files changed, 696 insertions(+), 542 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java index adadca7..8275f3f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterMultiTenantTableWithViewsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_MUTATE_TABLE; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT; import static org.junit.Assert.assertEquals; @@ -476,14 +477,14 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { // For a diverged view, only base table's pk column will be added and that too at the end. assertTableDefinition(conn, divergedView, PTableType.VIEW, baseTable, 2, 7, DIVERGED_VIEW_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2", "PK2"); - // Add existing column VIEW_COL2 to the base table + // Adding existing column VIEW_COL2 to the base table isn't allowed. alterBaseTable = "ALTER TABLE " + baseTable + " ADD VIEW_COL2 CHAR(256)"; - conn.createStatement().execute(alterBaseTable); - - // For the non-diverged view, adding the column VIEW_COL2 will end up changing its ordinal position in the view. - assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 2, 9, 8, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2", "VIEW_COL2", "VIEW_COL1"); - // For the diverged view, adding the column VIEW_COL2 will not change its ordinal position in the view. It also won't change the base column count or the sequence number - assertTableDefinition(conn, divergedView, PTableType.VIEW, baseTable, 2, 7, DIVERGED_VIEW_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2", "PK2"); + try { + conn.createStatement().execute(alterBaseTable); + fail(); + } catch (SQLException e) { + assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); + } } } @@ -500,13 +501,13 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { tenant1Conn.createStatement().execute(view1DDL); } - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 0, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 0, 7, 5, "TENANT_ID", "PK1", "V1", "V2", "V3", "VIEW_COL1", "VIEW_COL2"); String alterBaseTable = "ALTER TABLE " + baseTable + " ADD KV VARCHAR, PK2 VARCHAR PRIMARY KEY"; conn.createStatement().execute(alterBaseTable); - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 7, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 2, 7, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 1, 9, 7, "TENANT_ID", "PK1", "V1", "V2", "V3", "KV", "PK2", "VIEW_COL1", "VIEW_COL2"); // verify that the both columns were added to view1 @@ -530,13 +531,13 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { tenant1Conn.createStatement().execute(view1DDL); } - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 0, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 5, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V2", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 0, 7, 5, "TENANT_ID", "PK1", "V1", "V2", "V3", "VIEW_COL1", "VIEW_COL2"); String alterBaseTable = "ALTER TABLE " + baseTable + " DROP COLUMN V2"; conn.createStatement().execute(alterBaseTable); - assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 1, 4, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3"); + assertTableDefinition(conn, baseTable, PTableType.TABLE, null, 2, 4, BASE_TABLE_BASE_COLUMN_COUNT, "TENANT_ID", "PK1", "V1", "V3"); assertTableDefinition(conn, view1, PTableType.VIEW, baseTable, 1, 6, 4, "TENANT_ID", "PK1", "V1", "V3", "VIEW_COL1", "VIEW_COL2"); // verify that the dropped columns aren't visible @@ -569,7 +570,7 @@ public class AlterMultiTenantTableWithViewsIT extends ParallelStatsDisabledIT { String view1DDL = "CREATE VIEW " + view1 + " ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 CHAR(256)) AS SELECT * FROM " + baseTable; tenant1Conn.createStatement().execute(view1DDL); // This should not modify the base table - String alterView = "ALTER VIEW " + view1 + " ADD NEWCOL1 VARCHAR"; + String alterView = "ALTER VIEW " + view1 + " ADD NEWCOL1 VARCHAR, NEWCOL5 VARCHAR"; tenant1Conn.createStatement().execute(alterView); HTableDescriptor tableDesc2 = tenant1Conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin().getTableDescriptor(Bytes.toBytes(baseTable)); assertEquals(tableDesc1, tableDesc2); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java index 507108d..5841887 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java @@ -69,7 +69,6 @@ import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; /** @@ -2228,7 +2227,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { } @Test - public void testClientAndServerMetadataForEncodedColumns() throws Exception { + public void testMetadataForImmutableTableWithEncodedColumns() throws Exception { String schemaName = "XYZ"; String baseTableName = generateUniqueName(); String viewName = generateUniqueName(); @@ -2242,7 +2241,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { + " COL2 bigint NOT NULL," + " KV1 VARCHAR" + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)" - + " )"); + + " ) IMMUTABLE_ROWS = true"); PTable baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); long initBaseTableSeqNumber = baseTable.getSequenceNumber(); @@ -2278,6 +2277,54 @@ public class AlterTableIT extends ParallelStatsDisabledIT { } @Test + public void testMetadataForMutableTableWithEncodedColumns() throws Exception { + String schemaName = "XYZ"; + String baseTableName = generateUniqueName(); + String viewName = generateUniqueName(); + String fullTableName = schemaName + "." + baseTableName; + String fullViewName = schemaName + "." + viewName; + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + conn.createStatement().execute("CREATE TABLE IF NOT EXISTS " + fullTableName + " (" + + " ID char(1) NOT NULL," + + " COL1 integer NOT NULL," + + " COL2 bigint NOT NULL," + + " KV1 VARCHAR" + + " CONSTRAINT NAME_PK PRIMARY KEY (ID, COL1, COL2)" + + " )"); + PTable baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + long initBaseTableSeqNumber = baseTable.getSequenceNumber(); + + // assert that the client side cache is updated. + EncodedCQCounter cqCounter = baseTable.getEncodedCQCounter(); + assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 1), cqCounter.getValue(QueryConstants.DEFAULT_COLUMN_FAMILY)); + + + // assert that the server side metadata is updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1, true); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "KV1", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber); + + // now create a view and validate client and server side metadata + String viewDDL = "CREATE VIEW " + fullViewName + " ( VIEW_COL1 INTEGER, A.VIEW_COL2 VARCHAR ) AS SELECT * FROM " + fullTableName; + conn.createStatement().execute(viewDDL); + baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); + PTable view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); + + // verify that the client side cache is updated. Base table's cq counters should be updated. + assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 3), baseTable.getEncodedCQCounter().getValue(DEFAULT_COLUMN_FAMILY)); + assertNull("A view should always have the null cq counter", view.getEncodedCQCounter().getValue(DEFAULT_COLUMN_FAMILY)); + + // assert that the server side metadata for the base table and the view is also updated correctly. + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3, true); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); + assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + 1); + assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM); + } + } + + @Test public void testAddingColumnsToTablesAndViewsWithEncodedColumns() throws Exception { String schemaName = "XYZ"; String baseTableName = generateUniqueName(); @@ -2305,17 +2352,15 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // assert that the client side cache is updated. baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); EncodedCQCounter encodedCqCounter = baseTable.getEncodedCQCounter(); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 2), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 1), encodedCqCounter.getValue("B")); + assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 3), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); // assert that the server side metadata is updated correctly. - assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2, true); - assertEncodedCQCounter("B", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1, true); + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3, true); // assert that the server side metadata for columns is updated correctly. assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL4", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL5", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); - assertEncodedCQValue("B", "COL6", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertEncodedCQValue("B", "COL6", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + 1); // Create a view @@ -2323,13 +2368,11 @@ public class AlterTableIT extends ParallelStatsDisabledIT { conn.createStatement().execute(viewDDL); // assert that the server side metadata is updated correctly. - assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3, true); - assertEncodedCQCounter("A", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1, true); - assertEncodedCQCounter("B", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1, true); + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 5, true); // assert that the server side metadata for columns is updated correctly. - assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); - assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 4); // Creating a view that adds its own columns should increment the base table's sequence number too. assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + 2); @@ -2341,9 +2384,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // assert that the client cache for the base table is updated baseTable = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullTableName)); encodedCqCounter = baseTable.getEncodedCQCounter(); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 4), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 2), encodedCqCounter.getValue("A")); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 2), encodedCqCounter.getValue("B")); + assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 8), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); // assert client cache for view PTable view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); @@ -2351,14 +2392,12 @@ public class AlterTableIT extends ParallelStatsDisabledIT { assertNull("A view should always have the column qualifier counter as null", view.getEncodedCQCounter().getValue(DEFAULT_COLUMN_FAMILY)); // assert that the server side metadata for the base table and the view is also updated correctly. - assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 4, true); - assertEncodedCQCounter("A", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2, true); - assertEncodedCQCounter("B", schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2, true); - assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); - assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL3", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); - assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE); - assertEncodedCQValue("A", "VIEW_COL4", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); - assertEncodedCQValue("B", "VIEW_COL5", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 8, true); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL1", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 3); + assertEncodedCQValue("A", "VIEW_COL2", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 4); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "VIEW_COL3", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 5); + assertEncodedCQValue("A", "VIEW_COL4", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 6); + assertEncodedCQValue("B", "VIEW_COL5", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 7); // Adding a column to the should increment the base table's sequence number too since we update the cq counters for column families. assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + 3); assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM + 1); @@ -2370,9 +2409,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT { // assert that the client cache for the base table is updated encodedCqCounter = baseTable.getEncodedCQCounter(); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 5), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 3), encodedCqCounter.getValue("A")); - assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 2), encodedCqCounter.getValue("B")); + assertEquals((Integer)(ENCODED_CQ_COUNTER_INITIAL_VALUE + 10), encodedCqCounter.getValue(DEFAULT_COLUMN_FAMILY)); // assert client cache for view view = phxConn.getTable(new PTableKey(phxConn.getTenantId(), fullViewName)); @@ -2380,10 +2417,9 @@ public class AlterTableIT extends ParallelStatsDisabledIT { assertNull("A view should always have the column qualifier counter as null", view.getEncodedCQCounter().getValue(DEFAULT_COLUMN_FAMILY)); // assert that the server side metadata for the base table and the view is also updated correctly. - assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 5), true); - assertEncodedCQCounter("A", schemaName, baseTableName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 3), true); - assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL10", schemaName, viewName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 4)); - assertEncodedCQValue("A", "COL11", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 2); + assertEncodedCQCounter(DEFAULT_COLUMN_FAMILY, schemaName, baseTableName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 10), true); + assertEncodedCQValue(DEFAULT_COLUMN_FAMILY, "COL10", schemaName, viewName, (ENCODED_CQ_COUNTER_INITIAL_VALUE + 8)); + assertEncodedCQValue("A", "COL11", schemaName, viewName, ENCODED_CQ_COUNTER_INITIAL_VALUE + 9); assertSequenceNumber(schemaName, baseTableName, initBaseTableSeqNumber + 4); assertSequenceNumber(schemaName, viewName, PTable.INITIAL_SEQ_NUM + 2); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java index e4baa20..d0bc2c0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java @@ -91,7 +91,7 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT { // adding a new pk column and a new regular column conn.createStatement().execute("ALTER TABLE " + tableName + " ADD COL3 varchar(10) PRIMARY KEY, COL4 integer"); - assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "COL3", "COL4"); + assertTableDefinition(conn, tableName, PTableType.TABLE, null, 2, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "COL3", "COL4"); assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 1, 7, 5, "ID", "COL1", "COL2", "COL3", "COL4", "VIEW_COL1", "VIEW_COL2"); } } @@ -119,7 +119,7 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT { // drop two columns from the base table conn.createStatement().execute("ALTER TABLE " + tableName + " DROP COLUMN COL3, COL5"); - assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 4, + assertTableDefinition(conn, tableName, PTableType.TABLE, null, 2, 4, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "COL4"); assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 1, 6, 4, "ID", "COL1", "COL2", "COL4", "VIEW_COL1", "VIEW_COL2"); @@ -163,73 +163,80 @@ public class AlterTableWithViewsIT extends ParallelStatsDisabledIT { viewConn.commit(); try { - // should fail because there is already a view column with same name of different type - conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 char(10)"); + // adding a key value column to the base table that already exists in the view is not allowed + conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL4 DECIMAL, VIEW_COL2 VARCHAR(256)"); fail(); - } - catch (SQLException e) { - assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); - } - - try { - // should fail because there is already a view column with same name with different scale - conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 DECIMAL(10,1)"); - fail(); - } - catch (SQLException e) { - assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); - } - - try { - // should fail because there is already a view column with same name with different length - conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 DECIMAL(9,2)"); - fail(); - } - catch (SQLException e) { - assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); - } - - try { - // should fail because there is already a view column with different length - conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL2 VARCHAR"); - fail(); - } - catch (SQLException e) { + } catch (SQLException e) { assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); } - - // validate that there were no columns added to the table or view - assertTableDefinition(conn, tableName, PTableType.TABLE, null, 0, 3, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2"); - assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 0, 9, 3, "ID", "COL1", "COL2", "VIEW_COL1", "VIEW_COL2", "VIEW_COL3", "VIEW_COL4", "VIEW_COL5", "VIEW_COL6"); - - // should succeed - conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL4 DECIMAL, VIEW_COL2 VARCHAR(256)"); - assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "VIEW_COL4", "VIEW_COL2"); - assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 1, 9, 5, "ID", "COL1", "COL2", "VIEW_COL4", "VIEW_COL2", "VIEW_COL1", "VIEW_COL3", "VIEW_COL5", "VIEW_COL6"); - - // query table - ResultSet rs = stmt.executeQuery("SELECT * FROM " + tableName); - assertTrue(rs.next()); - assertEquals("view1", rs.getString("ID")); - assertEquals(12, rs.getInt("COL1")); - assertEquals(13, rs.getInt("COL2")); - assertEquals("view5", rs.getString("VIEW_COL2")); - assertEquals(17, rs.getInt("VIEW_COL4")); - assertFalse(rs.next()); - - // query view - rs = stmt.executeQuery("SELECT * FROM " + viewOfTable); - assertTrue(rs.next()); - assertEquals("view1", rs.getString("ID")); - assertEquals(12, rs.getInt("COL1")); - assertEquals(13, rs.getInt("COL2")); - assertEquals(14, rs.getInt("VIEW_COL1")); - assertEquals("view5", rs.getString("VIEW_COL2")); - assertEquals("view6", rs.getString("VIEW_COL3")); - assertEquals(17, rs.getInt("VIEW_COL4")); - assertEquals(18, rs.getInt("VIEW_COL5")); - assertEquals("view9", rs.getString("VIEW_COL6")); - assertFalse(rs.next()); +// try { +// // should fail because there is already a view column with same name of different type +// conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 char(10)"); +// fail(); +// } +// catch (SQLException e) { +// assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); +// } +// +// try { +// // should fail because there is already a view column with same name with different scale +// conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 DECIMAL(10,1)"); +// fail(); +// } +// catch (SQLException e) { +// assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); +// } +// +// try { +// // should fail because there is already a view column with same name with different length +// conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL1 DECIMAL(9,2)"); +// fail(); +// } +// catch (SQLException e) { +// assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); +// } +// +// try { +// // should fail because there is already a view column with different length +// conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL2 VARCHAR"); +// fail(); +// } +// catch (SQLException e) { +// assertEquals("Unexpected exception", CANNOT_MUTATE_TABLE.getErrorCode(), e.getErrorCode()); +// } +// +// // validate that there were no columns added to the table or view +// assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 3, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2"); +// assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 0, 9, 3, "ID", "COL1", "COL2", "VIEW_COL1", "VIEW_COL2", "VIEW_COL3", "VIEW_COL4", "VIEW_COL5", "VIEW_COL6"); +// +// // should succeed +// conn.createStatement().execute("ALTER TABLE " + tableName + " ADD VIEW_COL4 DECIMAL, VIEW_COL2 VARCHAR(256)"); +// assertTableDefinition(conn, tableName, PTableType.TABLE, null, 2, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "VIEW_COL4", "VIEW_COL2"); +// assertTableDefinition(conn, viewOfTable, PTableType.VIEW, tableName, 1, 9, 5, "ID", "COL1", "COL2", "VIEW_COL4", "VIEW_COL2", "VIEW_COL1", "VIEW_COL3", "VIEW_COL5", "VIEW_COL6"); +// +// // query table +// ResultSet rs = stmt.executeQuery("SELECT * FROM " + tableName); +// assertTrue(rs.next()); +// assertEquals("view1", rs.getString("ID")); +// assertEquals(12, rs.getInt("COL1")); +// assertEquals(13, rs.getInt("COL2")); +// assertEquals("view5", rs.getString("VIEW_COL2")); +// assertEquals(17, rs.getInt("VIEW_COL4")); +// assertFalse(rs.next()); +// +// // query view +// rs = stmt.executeQuery("SELECT * FROM " + viewOfTable); +// assertTrue(rs.next()); +// assertEquals("view1", rs.getString("ID")); +// assertEquals(12, rs.getInt("COL1")); +// assertEquals(13, rs.getInt("COL2")); +// assertEquals(14, rs.getInt("VIEW_COL1")); +// assertEquals("view5", rs.getString("VIEW_COL2")); +// assertEquals("view6", rs.getString("VIEW_COL3")); +// assertEquals(17, rs.getInt("VIEW_COL4")); +// assertEquals(18, rs.getInt("VIEW_COL5")); +// assertEquals("view9", rs.getString("VIEW_COL6")); +// assertFalse(rs.next()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java index 9ac7be3..002a7b1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java @@ -456,25 +456,25 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { assertTrue(rs.next()); assertEquals("A", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(12144, rs.getInt(3)); + assertEquals(12252, rs.getInt(3)); assertEquals(11, rs.getInt(4)); assertTrue(rs.next()); assertEquals("B", rs.getString(1)); assertEquals(20, rs.getInt(2)); - assertEquals(5540, rs.getInt(3)); + assertEquals(5600, rs.getInt(3)); assertEquals(5, rs.getInt(4)); assertTrue(rs.next()); assertEquals("C", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(6652, rs.getInt(3)); + assertEquals(6724, rs.getInt(3)); assertEquals(6, rs.getInt(4)); assertTrue(rs.next()); assertEquals("D", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(6652, rs.getInt(3)); + assertEquals(6724, rs.getInt(3)); assertEquals(6, rs.getInt(4)); assertFalse(rs.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java index bb13f1b..07fc4e9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; +import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_BYTES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -32,13 +34,22 @@ import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.expression.ArrayColumnExpression; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.tuple.ResultTuple; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -52,6 +63,8 @@ import org.junit.Test; * functionality allows having row-level versioning (similar to how KEEP_DELETED_CELLS works), but * also allows permanently deleting a row. */ + +//TODO: samarth parameterize this test once the storage scheme is optional public class StoreNullsIT extends ParallelStatsDisabledIT { private static final Log LOG = LogFactory.getLog(StoreNullsIT.class); @@ -95,31 +108,37 @@ public class StoreNullsIT extends ParallelStatsDisabledIT { } @Test - public void testStoringNulls() throws SQLException, InterruptedException, IOException { + public void testStoringNullsForImmutableTables() throws Exception { stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITH_NULLS + " VALUES (1, 'v1')"); stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITHOUT_NULLS + " VALUES (1, 'v1')"); stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITH_NULLS + " VALUES (2, null)"); stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITHOUT_NULLS + " VALUES (2, null)"); - ensureNullsNotStored(IMMUTABLE_WITH_NULLS); - ensureNullsNotStored(IMMUTABLE_WITHOUT_NULLS); + ensureNullsStoredAsEmptyByteArrays(IMMUTABLE_WITH_NULLS); + ensureNullsStoredAsEmptyByteArrays(IMMUTABLE_WITHOUT_NULLS); } - private void ensureNullsNotStored(String tableName) throws IOException { - tableName = SchemaUtil.normalizeIdentifier(tableName); + private void ensureNullsStoredAsEmptyByteArrays(String tableName) throws Exception { HTable htable = new HTable(getUtility().getConfiguration(), tableName); Scan s = new Scan(); s.setRaw(true); ResultScanner scanner = htable.getScanner(s); // first row has a value for name Result rs = scanner.next(); - assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("NAME"))); - assertTrue(rs.size() == 2); - // 2nd row has not + assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column + PColumn nameColumn = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, tableName)).getPColumnForColumnName("NAME"); + ArrayColumnExpression colExpression = new ArrayColumnExpression(nameColumn, "NAME", true); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + colExpression.evaluate(new ResultTuple(rs), ptr); + assertEquals(new ImmutableBytesPtr(PVarchar.INSTANCE.toBytes("v1")), ptr); + rs = scanner.next(); - assertFalse(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("NAME"))); - // and no delete marker either - assertTrue(rs.size() == 1); + assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column + + // assert null stored as empty + ptr = new ImmutableBytesPtr(); + colExpression.evaluate(new ResultTuple(rs), ptr); + assertEquals(new ImmutableBytesPtr(ByteUtil.EMPTY_BYTE_ARRAY), ptr); assertNull(scanner.next()); scanner.close(); htable.close(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java index 5d36a2f..b892c4d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java @@ -298,7 +298,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { ResultScanner results = table.getScanner(scan); Result result = results.next(); assertNotNull(result); - PTable viewIndexPTable = pconn.getTable(new PTableKey(pconn.getTenantId(), "VIEWINDEX2")); + PTable viewIndexPTable = pconn.getTable(new PTableKey(pconn.getTenantId(), viewIndex2)); PColumn column = viewIndexPTable.getPColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V4")); byte[] cq = EncodedColumnsUtil.getColumnQualifier(column, viewIndexPTable); // there should be a single row belonging to VIEWINDEX2 http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java index 7cca633..ee8afa8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java @@ -413,7 +413,12 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexTableName); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); - + + String sql = "SELECT LONG_COL1 from " + fullDataTableName + " WHERE LONG_COL2 = 2"; + rs = conn.createStatement().executeQuery(sql); + assertTrue(rs.next()); + assertFalse(rs.next()); + String dml = "DELETE from " + fullDataTableName + " WHERE long_col2 = 2"; assertEquals(1, conn.createStatement().executeUpdate(dml)); conn.commit(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index d770b22..e08e0d7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@ -276,6 +276,8 @@ public class MutableIndexFailureIT extends BaseOwnClusterIT { // verify index table has correct data validateDataWithIndex(conn, fullTableName, fullIndexName); validateDataWithIndex(conn, secondTableName, secondFullIndexName); + } finally { + FAIL_WRITE = false; } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java index 70b04d5..bcb12a9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java @@ -30,6 +30,7 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.ComparisonExpression; import org.apache.phoenix.expression.Determinism; import org.apache.phoenix.expression.Expression; @@ -221,7 +222,7 @@ public class CreateTableCompiler { } } - private static class ViewWhereExpressionVisitor extends StatelessTraverseNoExpressionVisitor<Boolean> { + public static class ViewWhereExpressionVisitor extends StatelessTraverseNoExpressionVisitor<Boolean> { private boolean isUpdatable = true; private final PTable table; private int position; @@ -307,6 +308,16 @@ public class CreateTableCompiler { return Boolean.TRUE; } + @Override + public Boolean visit(ArrayColumnExpression node) { + try { + this.position = table.getColumnFamily(node.getColumnFamily()).getPColumnForColumnQualifier(node.getEncodedColumnQualifier()).getPosition(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + return Boolean.TRUE; + } + } private static class VarbinaryDatum implements PDatum { http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java index bce94c2..846e966 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java @@ -400,7 +400,7 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expressio } protected void addColumn(PColumn column) { - context.getScan().addColumn(column.getFamilyName().getBytes(), column.getName().getBytes()); + EncodedColumnsUtil.setColumns(column, context.getCurrentTable().getTable(), context.getScan()); } @Override @@ -417,7 +417,7 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expressio return LiteralExpression.newConstant(column.getDataType().toObject(ptr), column.getDataType()); } if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(column)) { // project only kv columns - EncodedColumnsUtil.setColumns(column, tableRef.getTable(), context.getScan()); + addColumn(column); } Expression expression = ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive()); Expression wrappedExpression = wrapGroupByExpression(expression); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/compile/PostLocalIndexDDLCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostLocalIndexDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostLocalIndexDDLCompiler.java index 81dbe0d..d6fd6ff 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostLocalIndexDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostLocalIndexDDLCompiler.java @@ -31,6 +31,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.tuple.Tuple; @@ -82,7 +83,11 @@ public class PostLocalIndexDDLCompiler { // However, in this case, we need to project all of the data columns that contribute to the index. IndexMaintainer indexMaintainer = index.getIndexMaintainer(dataTable, connection); for (ColumnReference columnRef : indexMaintainer.getAllColumns()) { - scan.addColumn(columnRef.getFamily(), columnRef.getQualifier()); + if (index.getStorageScheme() == StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL) { + scan.addFamily(columnRef.getFamily()); + } else { + scan.addColumn(columnRef.getFamily(), columnRef.getQualifier()); + } } // Go through MutationPlan abstraction so that we can create local indexes http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 63ad9c0..601b50c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -229,21 +229,6 @@ public class WhereCompiler { } } - public void increment(ArrayColumnExpression column) { - switch (count) { - case NONE: - count = Count.SINGLE; - this.column = column.getArrayExpression(); - break; - case SINGLE: - count = column.getArrayExpression().equals(this.column) ? Count.SINGLE : Count.MULTIPLE; - break; - case MULTIPLE: - break; - - } - } - public Count getCount() { return count; } @@ -278,12 +263,6 @@ public class WhereCompiler { counter.increment(expression); return null; } - - @Override - public Void visit(ArrayColumnExpression expression) { - counter.increment(expression); - return null; - } }); switch (counter.getCount()) { case NONE: http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java index d21508a..677ebe8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java @@ -136,7 +136,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan); final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan); - boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan), j != null); + boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan)); if (ScanUtil.isLocalIndex(scan) || (j == null && p != null)) { if (dataColumns != null) { tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns); @@ -161,9 +161,9 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { } if (keyOrdered) { // Optimize by taking advantage that the rows are // already in the required group by key order - return scanOrdered(c, scan, innerScanner, expressions, aggregators, limit, j != null); + return scanOrdered(c, scan, innerScanner, expressions, aggregators, limit); } else { // Otherwse, collect them all up in an in memory map - return scanUnordered(c, scan, innerScanner, expressions, aggregators, limit, j != null); + return scanUnordered(c, scan, innerScanner, expressions, aggregators, limit); } } @@ -369,7 +369,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { */ private RegionScanner scanUnordered(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, final RegionScanner scanner, final List<Expression> expressions, - final ServerAggregators aggregators, long limit, boolean isJoin) throws IOException { + final ServerAggregators aggregators, long limit) throws IOException { if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations("Grouped aggregation over unordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators, ScanUtil.getCustomAnnotations(scan))); @@ -384,7 +384,7 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { (int) (Bytes.toInt(estDistValsBytes) * 1.5f)); } Pair<Integer, Integer> minMaxQualifiers = getMinMaxQualifiersFromScan(scan); - boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan), isJoin); + boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan)); final boolean spillableEnabled = conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE); @@ -450,14 +450,14 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver { */ private RegionScanner scanOrdered(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner scanner, final List<Expression> expressions, - final ServerAggregators aggregators, final long limit, final boolean isJoin) throws IOException { + final ServerAggregators aggregators, final long limit) throws IOException { if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations("Grouped aggregation over ordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators, ScanUtil.getCustomAnnotations(scan))); } final Pair<Integer, Integer> minMaxQualifiers = getMinMaxQualifiersFromScan(scan); - final boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(minMaxQualifiers, isJoin); + final boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(minMaxQualifiers); return new BaseRegionScanner(scanner) { private long rowCount = 0; private ImmutableBytesPtr currentKey = null; http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index a7247e2..adcff17 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -160,7 +160,6 @@ import org.apache.phoenix.expression.LiteralExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor; -import org.apache.phoenix.hbase.index.covered.update.ColumnReference; import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; @@ -1526,46 +1525,46 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso String tenantIdStr = tenantIdBytes.length == 0 ? null : Bytes.toString(tenantIdBytes); final Properties props = new Properties(); UpgradeUtil.doNotUpgradeOnFirstConnection(props); - try (PhoenixConnection connection = DriverManager.getConnection(MetaDataUtil.getJdbcUrl(env), props).unwrap(PhoenixConnection.class)){ - PName physicalName = parentTable.getPhysicalName(); - int nSequenceSaltBuckets = connection.getQueryServices().getSequenceSaltBuckets(); - SequenceKey key = MetaDataUtil.getViewIndexSequenceKey(tenantIdStr, physicalName, + try (PhoenixConnection connection = DriverManager.getConnection(MetaDataUtil.getJdbcUrl(env), props).unwrap(PhoenixConnection.class)) { + PName physicalName = parentTable.getPhysicalName(); + int nSequenceSaltBuckets = connection.getQueryServices().getSequenceSaltBuckets(); + SequenceKey key = MetaDataUtil.getViewIndexSequenceKey(tenantIdStr, physicalName, nSequenceSaltBuckets, parentTable.isNamespaceMapped() ); // TODO Review Earlier sequence was created at (SCN-1/LATEST_TIMESTAMP) and incremented at the client max(SCN,dataTable.getTimestamp), but it seems we should // use always LATEST_TIMESTAMP to avoid seeing wrong sequence values by different connection having SCN // or not. - long sequenceTimestamp = HConstants.LATEST_TIMESTAMP; - try { - connection.getQueryServices().createSequence(key.getTenantId(), key.getSchemaName(), key.getSequenceName(), + long sequenceTimestamp = HConstants.LATEST_TIMESTAMP; + try { + connection.getQueryServices().createSequence(key.getTenantId(), key.getSchemaName(), key.getSequenceName(), Short.MIN_VALUE, 1, 1, Long.MIN_VALUE, Long.MAX_VALUE, false, sequenceTimestamp); - } catch (SequenceAlreadyExistsException e) { - } - long[] seqValues = new long[1]; - SQLException[] sqlExceptions = new SQLException[1]; - connection.getQueryServices().incrementSequences(Collections.singletonList(new SequenceAllocation(key, 1)), + } catch (SequenceAlreadyExistsException e) { + } + long[] seqValues = new long[1]; + SQLException[] sqlExceptions = new SQLException[1]; + connection.getQueryServices().incrementSequences(Collections.singletonList(new SequenceAllocation(key, 1)), HConstants.LATEST_TIMESTAMP, seqValues, sqlExceptions); - if (sqlExceptions[0] != null) { - throw sqlExceptions[0]; - } - long seqValue = seqValues[0]; - if (seqValue > Short.MAX_VALUE) { - builder.setReturnCode(MetaDataProtos.MutationCode.TOO_MANY_INDEXES); - builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis()); - done.run(builder.build()); - return; - } - Put tableHeaderPut = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetadata); - NavigableMap<byte[], List<Cell>> familyCellMap = tableHeaderPut.getFamilyCellMap(); - List<Cell> cells = familyCellMap.get(TABLE_FAMILY_BYTES); - Cell cell = cells.get(0); - PDataType dataType = MetaDataUtil.getViewIndexIdDataType(); - Object val = dataType.toObject(seqValue, PLong.INSTANCE); - byte[] bytes = new byte [dataType.getByteSize() + 1]; - dataType.toBytes(val, bytes, 0); - Cell indexIdCell = new KeyValue(cell.getRow(), cell.getFamily(), VIEW_INDEX_ID_BYTES, + if (sqlExceptions[0] != null) { + throw sqlExceptions[0]; + } + long seqValue = seqValues[0]; + if (seqValue > Short.MAX_VALUE) { + builder.setReturnCode(MetaDataProtos.MutationCode.TOO_MANY_INDEXES); + builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis()); + done.run(builder.build()); + return; + } + Put tableHeaderPut = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetadata); + NavigableMap<byte[], List<Cell>> familyCellMap = tableHeaderPut.getFamilyCellMap(); + List<Cell> cells = familyCellMap.get(TABLE_FAMILY_BYTES); + Cell cell = cells.get(0); + PDataType dataType = MetaDataUtil.getViewIndexIdDataType(); + Object val = dataType.toObject(seqValue, PLong.INSTANCE); + byte[] bytes = new byte [dataType.getByteSize() + 1]; + dataType.toBytes(val, bytes, 0); + Cell indexIdCell = new KeyValue(cell.getRow(), cell.getFamily(), VIEW_INDEX_ID_BYTES, cell.getTimestamp(), Type.codeToType(cell.getTypeByte()), bytes); - cells.add(indexIdCell); - indexId = (short) seqValue; + cells.add(indexIdCell); + indexId = (short) seqValue; } } @@ -1576,7 +1575,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso // indexing on the system table. This is an issue because of the way we manage batch mutation // in the Indexer. region.mutateRowsWithLocks(tableMetadata, Collections.<byte[]> emptySet()); - + // Invalidate the cache - the next getTable call will add it // TODO: consider loading the table that was just created here, patching up the parent table, and updating the cache Cache<ImmutableBytesPtr,PMetaDataEntity> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache(); @@ -2609,6 +2608,22 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private MetaDataMutationResult validateColumnForAddToBaseTable(PColumn existingViewColumn, Put columnToBeAdded, PTable basePhysicalTable, boolean isColumnToBeAddPkCol, PTable view) { if (existingViewColumn != null) { if (EncodedColumnsUtil.usesEncodedColumnNames(basePhysicalTable) && !SchemaUtil.isPKColumn(existingViewColumn)) { + /* + * If the column already exists in a view, then we cannot add the column to the base + * table. The reason is subtle and is as follows: consider the case where a table + * has two views where both the views have the same key value column KV. Now, we + * dole out encoded column qualifiers for key value columns in views by using the + * counters stored in the base physical table. So the KV column can have different + * column qualifiers for the two views. For example, 11 for VIEW1 and 12 for VIEW2. + * This naturally extends to rows being inserted using the two views having + * different column qualifiers for the column named KV. Now, when an attempt is made + * to add column KV to the base table, we cannot decide which column qualifier + * should that column be assigned. It cannot be a number different than 11 or 12 + * since a query like SELECT KV FROM BASETABLE would return null for KV which is + * incorrect since column KV is present in rows inserted from the two views. We + * cannot use 11 or 12 either because we will then incorrectly return value of KV + * column inserted using only one view. + */ return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), basePhysicalTable); } // Validate data type is same @@ -2840,6 +2855,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso return mutationResult; } } + } else if (type == PTableType.VIEW + && EncodedColumnsUtil.usesEncodedColumnNames(table)) { + /* + * When adding a column to a view that uses encoded column name scheme, we + * need to modify the CQ counters stored in the view's physical table. So to + * make sure clients get the latest PTable, we need to invalidate the cache + * entry. + */ + invalidateList.add(new ImmutableBytesPtr(MetaDataUtil + .getPhysicalTableRowForView(table))); } for (Mutation m : tableMetaData) { byte[] key = m.getRow(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java index 5cfb102..e1ad6ea 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java @@ -107,7 +107,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { } } - private static OrderedResultIterator deserializeFromScan(Scan scan, RegionScanner s, boolean isJoin) { + private static OrderedResultIterator deserializeFromScan(Scan scan, RegionScanner s) { byte[] topN = scan.getAttribute(BaseScannerRegionObserver.TOPN); if (topN == null) { return null; @@ -125,7 +125,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { orderByExpression.readFields(input); orderByExpressions.add(orderByExpression); } - ResultIterator inner = new RegionScannerResultIterator(s, ScanUtil.getMinMaxQualifiersFromScan(scan), isJoin); + ResultIterator inner = new RegionScannerResultIterator(s, ScanUtil.getMinMaxQualifiersFromScan(scan)); return new OrderedResultIterator(inner, orderByExpressions, thresholdBytes, limit >= 0 ? limit : null, null, estimatedRowSize); } catch (IOException e) { @@ -219,7 +219,7 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan); final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan); //TODO: samarth get rid of this join shit. Joins should support position based look up. - boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan), j != null) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; + boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan)) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; innerScanner = getWrappedScanner(c, innerScanner, arrayKVRefs, arrayFuncRefs, offset, scan, dataColumns, tupleProjector, dataRegion, indexMaintainer, tx, @@ -231,10 +231,10 @@ public class ScanRegionObserver extends BaseScannerRegionObserver { } if (scanOffset != null) { innerScanner = getOffsetScanner(c, innerScanner, - new OffsetResultIterator(new RegionScannerResultIterator(innerScanner, ScanUtil.getMinMaxQualifiersFromScan(scan), j != null), scanOffset), + new OffsetResultIterator(new RegionScannerResultIterator(innerScanner, ScanUtil.getMinMaxQualifiersFromScan(scan)), scanOffset), scan.getAttribute(QueryConstants.LAST_SCAN) != null); } - final OrderedResultIterator iterator = deserializeFromScan(scan, innerScanner, j != null); + final OrderedResultIterator iterator = deserializeFromScan(scan, innerScanner); if (iterator == null) { return innerScanner; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 3129ef8..c39051d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -323,7 +323,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan); final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan); final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan); - boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan), j != null) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; + boolean useQualifierAsIndex = ScanUtil.useQualifierAsIndex(ScanUtil.getMinMaxQualifiersFromScan(scan)) && scan.getAttribute(BaseScannerRegionObserver.TOPN) != null; if ((localIndexScan && !isDelete && !isDescRowKeyOrderUpgrade) || (j == null && p != null)) { if (dataColumns != null) { tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns); @@ -498,7 +498,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver Put put = maintainer.buildUpdateMutation(kvBuilder, valueGetter, ptr, results.get(0).getTimestamp(), env.getRegion().getRegionInfo().getStartKey(), - env.getRegion().getRegionInfo().getEndKey(), false); + env.getRegion().getRegionInfo().getEndKey()); indexMutations.add(put); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java index 03b6c61..1a56735 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java @@ -23,88 +23,122 @@ import java.io.IOException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.expression.visitor.StatelessTraverseNoExpressionVisitor; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PVarbinary; -import org.apache.phoenix.schema.types.PVarchar; -import org.apache.phoenix.util.EncodedColumnsUtil; +import org.apache.phoenix.util.SchemaUtil; /** * * Class to access a column that is stored in a KeyValue that contains all - * columns for a given column family (stored in an array) + * columns for a given column family (stored in an array). * */ -public class ArrayColumnExpression extends ColumnExpression { +public class ArrayColumnExpression extends KeyValueColumnExpression { - private String displayName; // client-side only - private int index; - // expression that represents the array (where all cols are stored in a single key value) - private KeyValueColumnExpression arrayExpression; - // expression that represents this column if (it were stored as a regular key value) - private KeyValueColumnExpression origKVExpression; + private int encodedCQ; + private String displayName; public ArrayColumnExpression() { } - public ArrayColumnExpression(PDatum column, byte[] cf, int index) { - super(column); - this.index = index; - this.arrayExpression = new KeyValueColumnExpression(column, cf, cf); + public ArrayColumnExpression(PDatum column, byte[] cf, int encodedCQ) { + super(column, cf, cf); + this.encodedCQ = encodedCQ; } public ArrayColumnExpression(PColumn column, String displayName, boolean encodedColumnName) { - super(column); - // array indexes are 1-based TODO: samarth think about the case when the encodedcolumn qualifier is null. Probably best to add a check here for encodedcolumnname to be true - this.index = column.getEncodedColumnQualifier() + 1; - byte[] cf = column.getFamilyName().getBytes(); - this.arrayExpression = new KeyValueColumnExpression(column, cf, cf); - this.origKVExpression = new KeyValueColumnExpression(column, displayName, encodedColumnName); - this.displayName = displayName; + super(column, column.getFamilyName().getBytes(), column.getFamilyName().getBytes()); + this.displayName = SchemaUtil.getColumnDisplayName(column.getFamilyName().getString(), column.getName().getString()); + // TODO: samarth think about the case when the encodedcolumn qualifier is null. Probably best to add a check here for encodedcolumnname to be true + this.encodedCQ = column.getEncodedColumnQualifier(); } @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpression, PVarbinary.INSTANCE, null); - } + if (!super.evaluate(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; + } - @Override - public <T> T accept(ExpressionVisitor<T> visitor) { - return visitor.visit(this); + // Given a ptr to the entire array, set ptr to point to a particular element within that array + // given the type of an array element (see comments in PDataTypeForArray) + PArrayDataType.positionAtArrayElement(ptr, encodedCQ, PVarbinary.INSTANCE, null); + return true; } - + @Override public void readFields(DataInput input) throws IOException { super.readFields(input); - index = WritableUtils.readVInt(input); - arrayExpression = new KeyValueColumnExpression(); - arrayExpression.readFields(input); - origKVExpression = new KeyValueColumnExpression(); - origKVExpression.readFields(input); + encodedCQ = WritableUtils.readVInt(input); } @Override public void write(DataOutput output) throws IOException { super.write(output); - WritableUtils.writeVInt(output, index); - arrayExpression.write(output); - origKVExpression.write(output); - } - - public KeyValueColumnExpression getArrayExpression() { - return arrayExpression; + WritableUtils.writeVInt(output, encodedCQ); } public KeyValueColumnExpression getKeyValueExpression() { - return origKVExpression; + final boolean isNullable = isNullable(); + final SortOrder sortOrder = getSortOrder(); + final Integer scale = getScale(); + final Integer maxLength = getMaxLength(); + final PDataType datatype = getDataType(); + return new KeyValueColumnExpression(new PDatum() { + + @Override + public boolean isNullable() { + return isNullable; + } + + @Override + public SortOrder getSortOrder() { + return sortOrder; + } + + @Override + public Integer getScale() { + return scale; + } + + @Override + public Integer getMaxLength() { + return maxLength; + } + + @Override + public PDataType getDataType() { + return datatype; + } + }, getColumnFamily(), PInteger.INSTANCE.toBytes(encodedCQ)); } @Override public String toString() { return displayName; } - + + public byte[] getEncodedColumnQualifier() { + return PInteger.INSTANCE.toBytes(encodedCQ); + } + + @Override + public <T> T accept(ExpressionVisitor<T> visitor) { + //FIXME: this is ugly but can't think of a good solution. + if (visitor instanceof ViewWhereExpressionVisitor) { + return visitor.visit(this); + } else { + return super.accept(visitor); + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 35862c4..c6799d4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -122,7 +122,7 @@ public class KeyValueColumnExpression extends ColumnExpression { } @Override - public final <T> T accept(ExpressionVisitor<T> visitor) { + public <T> T accept(ExpressionVisitor<T> visitor) { return visitor.visit(this); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ReplaceArrayColumnWithKeyValueColumnExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ReplaceArrayColumnWithKeyValueColumnExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ReplaceArrayColumnWithKeyValueColumnExpressionVisitor.java deleted file mode 100644 index 7ca6d9e..0000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ReplaceArrayColumnWithKeyValueColumnExpressionVisitor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.expression.visitor; - -import java.util.List; - -import org.apache.phoenix.expression.ArrayColumnExpression; -import org.apache.phoenix.expression.Expression; - -public class ReplaceArrayColumnWithKeyValueColumnExpressionVisitor extends CloneExpressionVisitor { - - @Override - public boolean isCloneNode(Expression node, List<Expression> children) { - return !children.equals(node.getChildren()); - } - - @Override - public Expression visit(ArrayColumnExpression node) { - return node.getKeyValueExpression(); - } - -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java index 5909286..a7146fc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java @@ -188,11 +188,6 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil inputTuple.addColumn(expression.getColumnFamily(), expression.getColumnQualifier()); return null; } - @Override - public Void visit(ArrayColumnExpression expression) { - inputTuple.addColumn(expression.getArrayExpression().getColumnFamily(), expression.getArrayExpression().getColumnQualifier()); - return null; - } }; expression.accept(visitor); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1958d07/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java index 527b948..b97c4e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java @@ -63,12 +63,6 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi cq = expression.getColumnQualifier(); return null; } - @Override - public Void visit(ArrayColumnExpression expression) { - cf = expression.getArrayExpression().getColumnFamily(); - cq = expression.getArrayExpression().getColumnQualifier(); - return null; - } }; expression.accept(visitor); }