PHOENIX-4891: An index should inherit UPDATE_CACHE_FREQUENCY setting rom parent table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/64788794 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/64788794 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/64788794 Branch: refs/heads/omid2 Commit: 647887947f631d624d6897d729bce3acb07a94bf Parents: fe466eb Author: Chinmay Kulkarni <chinmayskulka...@gmail.com> Authored: Tue Oct 30 14:52:26 2018 -0700 Committer: Thomas D'Silva <tdsi...@apache.org> Committed: Tue Oct 30 16:23:42 2018 -0700 ---------------------------------------------------------------------- .../phoenix/end2end/PropertiesInSyncIT.java | 172 ++++++++++++++----- .../phoenix/end2end/index/IndexMetadataIT.java | 145 +++++++++++++++- .../org/apache/phoenix/rpc/UpdateCacheIT.java | 134 ++++++++++----- .../phoenix/exception/SQLExceptionCode.java | 7 +- .../query/ConnectionQueryServicesImpl.java | 6 +- .../apache/phoenix/schema/MetaDataClient.java | 33 +++- .../org/apache/phoenix/util/MetaDataUtil.java | 6 +- .../org/apache/phoenix/util/UpgradeUtil.java | 79 ++++++++- 8 files changed, 476 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java index db44735..348b195 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java @@ -23,9 +23,12 @@ import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; @@ -33,17 +36,23 @@ import org.junit.Test; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Properties; import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; -import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES; +import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES; import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX; +import static org.apache.phoenix.util.UpgradeUtil.UPSERT_UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.util.UpgradeUtil.syncTableAndIndexProperties; +import static org.apache.phoenix.util.UpgradeUtil.syncUpdateCacheFreqAllIndexes; +import static org.apache.phoenix.end2end.index.IndexMetadataIT.assertUpdateCacheFreq; /** * Test properties that need to be kept in sync amongst all column families and indexes of a table @@ -56,12 +65,16 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { private static final int INITIAL_TTL_VALUE = 700; private static final KeepDeletedCells INITIAL_KEEP_DELETED_CELLS_VALUE = KeepDeletedCells.TRUE; private static final int INITIAL_REPLICATION_SCOPE_VALUE = 1; + private static final int INITIAL_UPDATE_CACHE_FREQUENCY = 100; + private static final int INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS = 900; private static final int MODIFIED_TTL_VALUE = INITIAL_TTL_VALUE + 300; private static final KeepDeletedCells MODIFIED_KEEP_DELETED_CELLS_VALUE = - (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE) - ? KeepDeletedCells.FALSE: KeepDeletedCells.TRUE; + (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE) ? + KeepDeletedCells.FALSE: KeepDeletedCells.TRUE; private static final int MODIFIED_REPLICATION_SCOPE_VALUE = (INITIAL_REPLICATION_SCOPE_VALUE == 1) ? 0 : 1; + private static final int MODIFIED_UPDATE_CACHE_FREQUENCY = INITIAL_UPDATE_CACHE_FREQUENCY + 300; + private static final int MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS = INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS + 300; // Test that we disallow specifying synced properties to be set per column family @@ -70,7 +83,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { public void testDisallowSyncedPropsToBeSetColFamSpecificCreateTable() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), new Properties()); String tableName = generateUniqueName(); - for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { try { conn.createStatement().execute("create table " + tableName + " (id INTEGER not null primary key, " @@ -107,7 +120,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { String viewName = "VIEW_" + tableName; conn.createStatement().execute("create view " + viewName + " (new_col SMALLINT) as select * from " + tableName + " where id > 1"); - for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { try { conn.createStatement().execute("create local index " + localIndexName + " on " + tableName + "(name) " @@ -148,8 +161,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { public void testSyncedPropsBaseTableCreateIndex() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), new Properties()); String tableName = createBaseTableWithProps(conn); - createIndexTable(conn, tableName, PTable.IndexType.LOCAL); - String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL); + createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond(); + String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond(); // We pass the base table as the physical HBase table since our check includes checking // the local index column family too @@ -164,7 +177,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { public void testSyncedPropsBaseTableCreateViewIndex() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), new Properties()); String tableName = createBaseTableWithProps(conn); - String viewIndexName = createIndexTable(conn, tableName, null); + String viewIndexName = createIndexTable(conn, tableName, null).getSecond(); verifyHBaseColumnFamilyProperties(tableName, conn, false, false); verifyHBaseColumnFamilyProperties(viewIndexName, conn, false, false); @@ -179,7 +192,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { String tableName = createBaseTableWithProps(conn); StringBuilder alterAllSyncedPropsString = new StringBuilder(); String modPropString = COL_FAM1 + ".%s=" + DUMMY_PROP_VALUE + ","; - for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { try { conn.createStatement().execute("alter table " + tableName + " set " + COL_FAM1 + "." + propName + "=" + DUMMY_PROP_VALUE); @@ -216,11 +229,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { Set<String> tablesToCheck = new HashSet<>(); tablesToCheck.add(tableName); for (int i=0; i<2; i++) { - tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL)); - tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL)); + tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond()); + tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond()); } // Create a view and view index - tablesToCheck.add(createIndexTable(conn, tableName, null)); + tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond()); // Now alter the base table's properties. This should get propagated to all index tables conn.createStatement().execute("alter table " + tableName + " set TTL=" + MODIFIED_TTL_VALUE @@ -232,8 +245,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { } // Any indexes created henceforth should have the modified properties - String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL); - String newViewIndex = createIndexTable(conn, tableName, null); + String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond(); + String newViewIndex = createIndexTable(conn, tableName, null).getSecond(); verifyHBaseColumnFamilyProperties(newGlobalIndex, conn, true, false); verifyHBaseColumnFamilyProperties(newViewIndex, conn, true, false); conn.close(); @@ -245,9 +258,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { Connection conn = DriverManager.getConnection(getUrl(), new Properties()); String tableName = createBaseTableWithProps(conn); - // Test that we are not allowed to set any property to be kept in sync, specific - // to the new column family to be added - for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + // Test that we are not allowed to set any property to be kept in sync, specific to the new column family to be added + for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { try { conn.createStatement().execute( "alter table " + tableName + " add " + NEW_CF + ".new_column varchar(2) " @@ -268,11 +280,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { Set<String> tablesToCheck = new HashSet<>(); tablesToCheck.add(tableName); for (int i=0; i<2; i++) { - tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL)); - tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL)); + tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond()); + tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond()); } // Create a view and view index - tablesToCheck.add(createIndexTable(conn, tableName, null)); + tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond()); // Now add a new column family while simultaneously modifying properties to be kept in sync, // as well as a property which does not need to be kept in sync. Properties to be kept @@ -307,8 +319,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { public void testDisallowAlterGlobalIndexTable() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), new Properties()); String tableName = createBaseTableWithProps(conn); - String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL); - for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond(); + for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { try { conn.createStatement().execute("alter table " + globalIndexName + " set " + propName + "=" + DUMMY_PROP_VALUE); @@ -336,12 +348,12 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { createdTables.add(baseTableName1); // Create different indexes on the base table for (int i=0; i<2; i++) { - createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL)); - createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL)); - createdTables.add(createIndexTable(conn, baseTableName, null)); - createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL)); - createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL)); - createdTables.add(createIndexTable(conn, baseTableName1, null)); + createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond()); + createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getSecond()); + createdTables.add(createIndexTable(conn, baseTableName, null).getSecond()); + createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL).getSecond()); + createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL).getSecond()); + createdTables.add(createIndexTable(conn, baseTableName1, null).getSecond()); } for (String t: createdTables) { verifyHBaseColumnFamilyProperties(t, conn, false, false); @@ -394,6 +406,77 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { conn.close(); } + @Test + public void testOldClientSyncUpdateCacheFreqUpgradePath() throws Exception { + PTable base, index; + String baseTableName, viewName, viewName2; + Map<String, Set<String>> createdTablesAndViews = new HashMap<>(); + + try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) { + baseTableName = createBaseTableWithProps(conn); + createdTablesAndViews.put(baseTableName, new HashSet<String>()); + Set<String> indexes = createdTablesAndViews.get(baseTableName); + indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond()); + indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getFirst()); + + viewName = createViewOnBaseTableOrView(conn, baseTableName); + createdTablesAndViews.put(viewName, new HashSet<String>()); + indexes = createdTablesAndViews.get(viewName); + indexes.add(createIndexTable(conn, viewName, PTable.IndexType.GLOBAL).getSecond()); + + viewName2 = createViewOnBaseTableOrView(conn, viewName); + createdTablesAndViews.put(viewName2, new HashSet<String>()); + indexes = createdTablesAndViews.get(viewName2); + indexes.add(createIndexTable(conn, viewName2, PTable.IndexType.LOCAL).getFirst()); + + // Intentionally make UPDATE_CACHE_FREQUENCY out of sync for indexes + PreparedStatement stmt = conn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY); + for (String tableOrViewName : createdTablesAndViews.keySet()) { + base = PhoenixRuntime.getTable(conn, tableOrViewName); + for (String indexTableName : createdTablesAndViews.get(tableOrViewName)) { + index = PhoenixRuntime.getTable(conn, indexTableName); + PName tenantId = index.getTenantId(); + stmt.setString(1, tenantId == null ? null : tenantId.getString()); + stmt.setString(2, index.getSchemaName().getString()); + stmt.setString(3, index.getTableName().getString()); + stmt.setLong(4, base.getType() == PTableType.TABLE ? + MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS); + stmt.addBatch(); + } + } + stmt.executeBatch(); + conn.commit(); + + // Clear the server-side cache so that we get the latest built PTables + conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + // Verify that the modified values are reflected + for (String tableOrViewName : createdTablesAndViews.keySet()) { + assertUpdateCacheFreq(conn, tableOrViewName, baseTableName.equals(tableOrViewName) ? + INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS); + for (String indexName : createdTablesAndViews.get(tableOrViewName)) { + assertUpdateCacheFreq(conn, indexName, baseTableName.equals(tableOrViewName) ? + MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS); + } + } + + PhoenixConnection upgradeConn = conn.unwrap(PhoenixConnection.class); + upgradeConn.setRunningUpgrade(true); + syncUpdateCacheFreqAllIndexes(upgradeConn, + PhoenixRuntime.getTableNoCache(conn, baseTableName)); + + conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + // Verify that indexes have the synced values for UPDATE_CACHE_FREQUENCY + for (String tableOrViewName : createdTablesAndViews.keySet()) { + long expectedVal = baseTableName.equals(tableOrViewName) ? + INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS; + assertUpdateCacheFreq(conn, tableOrViewName, expectedVal); + for (String indexOnTableOrView : createdTablesAndViews.get(tableOrViewName)) { + assertUpdateCacheFreq(conn, indexOnTableOrView, expectedVal); + } + } + } + } + /** * Helper method to modify the synced properties for a column family descriptor * @param cfd The column family descriptor object @@ -417,9 +500,9 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { conn.createStatement().execute("create table " + tableName + " (id INTEGER not null primary key, type varchar(5), " + COL_FAM1 + ".name varchar(10), " + COL_FAM2 + ".flag boolean) " - + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS=" - + INITIAL_KEEP_DELETED_CELLS_VALUE - + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE); + + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS=" + INITIAL_KEEP_DELETED_CELLS_VALUE + + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE + + ",UPDATE_CACHE_FREQUENCY=" + INITIAL_UPDATE_CACHE_FREQUENCY); return tableName; } @@ -429,34 +512,43 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT { * @param baseTableName Name of the HBase base table on which to create an index * @param indexType LOCAL, GLOBAL or if we pass in null as the indexType, * we create a view and an index on that view for the given base table - * @return The physical HBase table corresponding to the index created + * @return A pair consisting of the index name and the name of the physical HBase table + * corresponding to the index created * @throws SQLException */ - private String createIndexTable(Connection conn, String baseTableName, + private Pair<String,String> createIndexTable(Connection conn, String baseTableName, PTable.IndexType indexType) throws SQLException { // Create a view on top of the base table and then an index on that view if (indexType == null) { - String viewName = "VIEW_" + baseTableName + "_" + generateUniqueName(); + String viewName = createViewOnBaseTableOrView(conn, baseTableName); String viewIndexName = VIEW_INDEX_TABLE_PREFIX + baseTableName; - conn.createStatement().execute("create view " + viewName - + " (new_col SMALLINT) as select * from " + baseTableName + " where id > 1"); conn.createStatement().execute("create index view_index_" + generateUniqueName() + " on " + viewName + " (flag)"); - return viewIndexName; + return new Pair<>(viewIndexName, viewIndexName); } switch(indexType) { case LOCAL: String localIndexName = baseTableName + "_LOCAL_" + generateUniqueName(); conn.createStatement().execute( "create local index " + localIndexName + " on " + baseTableName + "(flag)"); - return baseTableName; + return new Pair<>(localIndexName, baseTableName); case GLOBAL: String globalIndexName = baseTableName + "_GLOBAL_" + generateUniqueName(); conn.createStatement() .execute("create index " + globalIndexName + " on " + baseTableName + "(name)"); - return globalIndexName; + return new Pair<>(globalIndexName, globalIndexName); + default: + return new Pair<>(baseTableName, baseTableName); } - return baseTableName; + } + + private String createViewOnBaseTableOrView(Connection conn, String baseTableOrView) throws SQLException { + String viewName = "VIEW_" + baseTableOrView + "_" + generateUniqueName(); + conn.createStatement().execute("create view " + viewName + + " (" + generateUniqueName() + " SMALLINT) as select * from " + + baseTableOrView + " where id > 1 UPDATE_CACHE_FREQUENCY=" + + INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS); + return viewName; } /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java index 21fa312..f797b70 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java @@ -19,6 +19,8 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -675,15 +677,12 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT { } } - - @Test public void testIndexAlterPhoenixProperty() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); String testTable = generateUniqueName(); - String ddl = "create table " + testTable + " (k varchar primary key, v1 varchar)"; Statement stmt = conn.createStatement(); stmt.execute(ddl); @@ -703,6 +702,123 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT { assertEquals(20,rs.getInt(1)); } + @Test + public void testCreateIndexSetUpdateCacheFreqFails() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + String testTable = generateUniqueName(); + + String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)"; + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + String indexName = "IDX_" + generateUniqueName(); + + ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) " + + "UPDATE_CACHE_FREQUENCY=10000"; + try { + stmt.execute(ddl); + fail("Should fail trying to set UPDATE_CACHE_FREQUENCY when creating an index"); + } catch (SQLException sqlE) { + assertEquals("Unexpected error occurred", + CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode()); + } + } + + @Test + public void testIndexGetsUpdateCacheFreqFromBaseTable() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + String testTable = generateUniqueName(); + + long updateCacheFreq = 10000; + String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) " + + "UPDATE_CACHE_FREQUENCY=" + updateCacheFreq; + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + + String localIndex = "LOCAL_" + generateUniqueName(); + String globalIndex = "GLOBAL_" + generateUniqueName(); + + ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + + // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value + assertUpdateCacheFreq(conn, testTable, updateCacheFreq); + assertUpdateCacheFreq(conn, localIndex, updateCacheFreq); + assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq); + } + + @Test + public void testAlterTablePropagatesUpdateCacheFreqToIndexes() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + String testTable = generateUniqueName(); + + String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) "; + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + + String localIndex = "LOCAL_" + generateUniqueName(); + String globalIndex = "GLOBAL_" + generateUniqueName(); + + ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + + assertUpdateCacheFreq(conn, testTable, DEFAULT_UPDATE_CACHE_FREQUENCY); + assertUpdateCacheFreq(conn, localIndex, DEFAULT_UPDATE_CACHE_FREQUENCY); + assertUpdateCacheFreq(conn, globalIndex, DEFAULT_UPDATE_CACHE_FREQUENCY); + + // Alter UPDATE_CACHE_FREQUENCY on the base table + long updateCacheFreq = 10000; + ddl = "ALTER TABLE " + testTable + " SET UPDATE_CACHE_FREQUENCY=" + updateCacheFreq; + stmt.execute(ddl); + + // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value + assertUpdateCacheFreq(conn, testTable, updateCacheFreq); + assertUpdateCacheFreq(conn, localIndex, updateCacheFreq); + assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq); + } + + @Test + public void testIndexAlterUpdateCacheFreqFails() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + String testTable = generateUniqueName(); + + String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)"; + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + + String localIndex = "LOCAL_" + generateUniqueName(); + String globalIndex = "GLOBAL_" + generateUniqueName(); + + ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) "; + stmt.execute(ddl); + + try { + stmt.execute("ALTER INDEX " + localIndex + " ON " + testTable + + " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER"); + fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index"); + } catch (SQLException sqlE) { + assertEquals("Unexpected error occurred", + CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode()); + } + + try { + stmt.execute("ALTER INDEX " + globalIndex + " ON " + testTable + + " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER"); + fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index"); + } catch (SQLException sqlE) { + assertEquals("Unexpected error occurred", + CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode()); + } + } @Test public void testIndexAlterHBaseProperty() throws Exception { @@ -718,10 +834,10 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT { ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) "; stmt.execute(ddl); - conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false"); - asssertIsWALDisabled(conn,indexName,false); conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=true"); asssertIsWALDisabled(conn,indexName,true); + conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false"); + asssertIsWALDisabled(conn,indexName,false); } private static void asssertIsWALDisabled(Connection conn, String fullTableName, boolean expectedValue) throws SQLException { @@ -729,4 +845,23 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT { assertEquals(expectedValue, pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).isWALDisabled()); } + /** + * Helper method to assert the value of UPDATE_CACHE_FREQUENCY for a table/index/view + * @param conn Phoenix connection + * @param name table/view/index name + * @param expectedUpdateCacheFreq expected value of UPDATE_CACHE_FREQUENCY + * @throws SQLException + */ + public static void assertUpdateCacheFreq(Connection conn, String name, + long expectedUpdateCacheFreq) throws SQLException { + ResultSet rs = conn.createStatement().executeQuery( + "select UPDATE_CACHE_FREQUENCY from SYSTEM.\"CATALOG\" where TABLE_NAME='" + + name + "'"); + assertTrue(rs.next()); + assertEquals("Mismatch found for " + name, expectedUpdateCacheFreq, rs.getLong(1)); + assertEquals("Mismatch in UPDATE_CACHE_FREQUENCY for PTable of " + name, + expectedUpdateCacheFreq, PhoenixRuntime.getTableNoCache(conn, name) + .getUpdateCacheFrequency()); + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java index 0ddbed3..2959b99 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java @@ -19,6 +19,7 @@ package org.apache.phoenix.rpc; import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.eq; @@ -73,7 +74,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName; Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)); conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + provider + "'"); - helpTestUpdateCache(fullTableName, new int[] {1, 1}); + helpTestUpdateCache(fullTableName, new int[] {1, 1}, false); } } } @@ -84,14 +85,14 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName; Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)); conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA); - helpTestUpdateCache(fullTableName, new int[] {1, 3}); + helpTestUpdateCache(fullTableName, new int[] {1, 3}, false); } @Test public void testUpdateCacheForNonTxnSystemTable() throws Exception { String fullTableName = "\""+ QueryConstants.SYSTEM_SCHEMA_NAME + "\""+ QueryConstants.NAME_SEPARATOR + generateUniqueName(); setupSystemTable(fullTableName); - helpTestUpdateCache(fullTableName, new int[] {0, 0}); + helpTestUpdateCache(fullTableName, new int[] {0, 0}, false); } @Test @@ -104,7 +105,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { conn.createStatement().execute( "alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER"); } - helpTestUpdateCache(fullTableName, new int[] {0, 0}); + helpTestUpdateCache(fullTableName, new int[] {0, 0}, false); } @Test @@ -114,7 +115,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always"); } - helpTestUpdateCache(fullTableName, new int[] {1, 3}); + helpTestUpdateCache(fullTableName, new int[] {1, 3}, false); } @Test @@ -124,9 +125,9 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000); } - helpTestUpdateCache(fullTableName, new int[] {0, 0}); + helpTestUpdateCache(fullTableName, new int[] {0, 0}, false); Thread.sleep(10000); - helpTestUpdateCache(fullTableName, new int[] {1, 0}); + helpTestUpdateCache(fullTableName, new int[] {1, 0}, false); } @Test @@ -136,14 +137,55 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never"); } - helpTestUpdateCache(fullTableName, new int[] {0, 0}); + helpTestUpdateCache(fullTableName, new int[] {0, 0}, false); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS"); } - helpTestUpdateCache(fullTableName, new int[] {1, 3}); + helpTestUpdateCache(fullTableName, new int[] {1, 3}, false); + } + + @Test + public void testUpdateCacheFreqPropagatedToIndexes() throws Exception { + String baseTableName = generateUniqueName(); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + baseTableName; + String localIndex = "LOCAL_" + baseTableName; + String globalIndex = "GLOBAL_" + baseTableName; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never"); + + // Create local and global indexes on the base table + conn.createStatement().execute("CREATE LOCAL INDEX " + localIndex + + " on " + fullTableName + " (a.date1, b.varchar_col2)"); + conn.createStatement().execute("CREATE INDEX " + globalIndex + " on " + + fullTableName + " (a.int_col1, a.long_col1)"); + } + + // The indexes should have got the UPDATE_CACHE_FREQUENCY value of their base table + helpTestUpdateCache(fullTableName, new int[] {0, 0}, false); + helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex, + new int[] {0}, true); + helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex, + new int[] {0}, true); + + // Now alter the UPDATE_CACHE_FREQUENCY value of the base table + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement() + .execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS"); + } + // Even the indexes should now have the modified value of UPDATE_CACHE_FREQUENCY + // Note that when we query the base table, during query plan generation, we make 2 getTable + // requests (to retrieve the base table) for each index of the base table + helpTestUpdateCache(fullTableName, new int[] {1, 15}, false); + helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex, + new int[] {3}, true); + helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex, + new int[] {3}, true); } - private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs) throws Exception { + private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs, + boolean skipUpsertForIndexes) throws Exception { String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); String selectSql = "SELECT * FROM "+fullTableName; @@ -154,46 +196,31 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { Connection conn = connectionQueryServices.connect(getUrl(), props); try { conn.setAutoCommit(false); - String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)"; - PreparedStatement stmt = conn.prepareStatement(upsert); - // upsert three rows - TestUtil.setRowKeyColumns(stmt, 1); - stmt.execute(); - TestUtil.setRowKeyColumns(stmt, 2); - stmt.execute(); - TestUtil.setRowKeyColumns(stmt, 3); - stmt.execute(); - conn.commit(); - int numUpsertRpcs = expectedRPCs[0]; - // verify only 0 or 1 rpc to fetch table metadata, - verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(), - eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), - anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull()); - reset(connectionQueryServices); - - ResultSet rs = conn.createStatement().executeQuery(selectSql); - TestUtil.validateRowKeyColumns(rs, 1); - TestUtil.validateRowKeyColumns(rs, 2); - TestUtil.validateRowKeyColumns(rs, 3); - assertFalse(rs.next()); - - rs = conn.createStatement().executeQuery(selectSql); - TestUtil.validateRowKeyColumns(rs, 1); - TestUtil.validateRowKeyColumns(rs, 2); - TestUtil.validateRowKeyColumns(rs, 3); - assertFalse(rs.next()); - - rs = conn.createStatement().executeQuery(selectSql); - TestUtil.validateRowKeyColumns(rs, 1); - TestUtil.validateRowKeyColumns(rs, 2); - TestUtil.validateRowKeyColumns(rs, 3); - assertFalse(rs.next()); - + if (!skipUpsertForIndexes) { + String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(upsert); + // upsert three rows + for (int i=0; i<3; i++) { + TestUtil.setRowKeyColumns(stmt, i); + stmt.execute(); + } + conn.commit(); + int numUpsertRpcs = expectedRPCs[0]; + // verify only 0 or 1 rpc to fetch table metadata, + verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(), + eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), + anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull()); + reset(connectionQueryServices); + } + validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes); + validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes); + validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes); + // for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query // for non-transactional tables with a scn : verify *only* one rpc occurs // for transactional tables : verify *only* one rpc occurs // for non-transactional, system tables : verify no rpc occurs - int numRpcs = expectedRPCs[1]; + int numRpcs = skipUpsertForIndexes ? expectedRPCs[0] : expectedRPCs[1]; verify(connectionQueryServices, times(numRpcs)).getTable((PName) isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull()); @@ -202,4 +229,19 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT { conn.close(); } } + + private static void validateSelectRowKeyCols(Connection conn, String selectSql, + boolean skipUpsertForIndexes) throws SQLException { + ResultSet rs = conn.createStatement().executeQuery(selectSql); + if (skipUpsertForIndexes) { + for (int i=0; i<3; i++) { + assertTrue(rs.next()); + } + } else { + for (int i=0; i<3; i++) { + TestUtil.validateRowKeyColumns(rs, i); + } + } + assertFalse(rs.next()); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index 6696521..5bffed5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.hbase.index.util.IndexManagementUtil; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.AmbiguousColumnException; @@ -263,7 +264,7 @@ public enum SQLExceptionCode { NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."), COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."), COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY(1050, "43A07", "Setting or altering any of the following properties: " - + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString() + + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString() + " for a column family is not supported since they must be kept in sync. You can only set these properties for the entire table."), CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."), CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."), @@ -308,7 +309,9 @@ public enum SQLExceptionCode { TTL_UNSUPPORTED_FOR_TXN_TABLE(10947, "44A28", "TTL is not supported for"), CANNOT_CREATE_LOCAL_INDEX_FOR_TXN_TABLE(10948, "44A29", "Local indexes cannot be created for"), CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX(10949, "44A30", "Cannot set or alter the following properties on an index: " - + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()), + + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()), + CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX(10950, "44A31", "Cannot set or alter " + + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " on an index"), /** Sequence related */ SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index ebcb7b9..1381247 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -2519,7 +2519,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement * Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families * in sync with the existing column families. Note that we use the new values for these properties in case they * are passed from our alter table command, if not, we use the default column family's value for each property - * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} * @param allFamiliesProps Map of all column family properties * @param table original table * @param tableDesc new table descriptor @@ -2565,7 +2565,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement /** * Set the new values for properties that are to be kept in sync amongst those column families of the table which are * not referenced in the context of our alter table command, including the local index column family if it exists - * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} * @param tableDesc original table descriptor * @param allFamiliesProps Map of all column family properties * @param newTTL new value of TTL @@ -2587,7 +2587,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement /** * Set properties to be kept in sync for global indexes of a table, as well as * the physical table corresponding to indexes created on views of a table - * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} and + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} and * @param table base table * @param tableAndIndexDescriptorMappings old to new table descriptor mappings * @param applyPropsToAllIndexesDefaultCF new properties to apply to all index column families http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 6563c40..df0e5d4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -272,6 +272,7 @@ public class MetaDataClient { TABLE_NAME + "," + SYNC_INDEX_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() + ") VALUES (?, ?, ?, ?)"; + private static final String CREATE_TABLE = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + @@ -1147,7 +1148,7 @@ public class MetaDataClient { /** * Populate properties for the table and common properties for all column families of the table * @param statementProps Properties specified in SQL statement - * @param tableProps Properties for an HTableDescriptor + * @param tableProps Properties for an HTableDescriptor and Phoenix Table Properties * @param commonFamilyProps Properties common to all column families * @param tableType Used to distinguish between index creation vs. base table creation paths * @throws SQLException @@ -1164,9 +1165,17 @@ public class MetaDataClient { .setMessage("Property: " + prop.getFirst()).build() .buildException(); } + // HTableDescriptor property or Phoenix Table Property if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) { + // See PHOENIX-4891 + if (tableType == PTableType.INDEX && UPDATE_CACHE_FREQUENCY.equals(prop.getFirst())) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX) + .build() + .buildException(); + } tableProps.put(prop.getFirst(), prop.getSecond()); - } else { + } else { // HColumnDescriptor property commonFamilyProps.put(prop.getFirst(), prop.getSecond()); } } @@ -2127,8 +2136,11 @@ public class MetaDataClient { } long updateCacheFrequency = connection.getQueryServices().getProps().getLong( QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY); + if (tableType == PTableType.INDEX && parent != null) { + updateCacheFrequency = parent.getUpdateCacheFrequency(); + } Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps); - if (updateCacheFrequencyProp != null) { + if (tableType != PTableType.INDEX && updateCacheFrequencyProp != null) { updateCacheFrequency = updateCacheFrequencyProp; } String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps); @@ -3613,9 +3625,13 @@ public class MetaDataClient { } } - if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || metaProperties.getNonTxToTx())) { + if (!table.getIndexes().isEmpty() && + (numPkColumnsAdded>0 || metaProperties.getNonTxToTx() || + metaPropertiesEvaluated.getUpdateCacheFrequency() != null)) { for (PTable index : table.getIndexes()) { - incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, metaProperties.getNonTxToTx() ? Boolean.TRUE : null, metaPropertiesEvaluated.getUpdateCacheFrequency()); + incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, + metaProperties.getNonTxToTx() ? Boolean.TRUE : null, + metaPropertiesEvaluated.getUpdateCacheFrequency()); } tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); @@ -4573,6 +4589,13 @@ public class MetaDataClient { } if (metaProperties.getUpdateCacheFrequencyProp() != null) { + // See PHOENIX-4891 + if (table.getType() == PTableType.INDEX) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX) + .build() + .buildException(); + } if (metaProperties.getUpdateCacheFrequencyProp().longValue() != table.getUpdateCacheFrequency()) { metaPropertiesEvaluated.setUpdateCacheFrequency(metaProperties.getUpdateCacheFrequencyProp()); changingPhoenixTableProperty = true; http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index df7530c..8cf514f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -109,7 +109,7 @@ public class MetaDataUtil { public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME); // See PHOENIX-3955 - public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES = ImmutableList.of( + public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES = ImmutableList.of( HColumnDescriptor.TTL, HColumnDescriptor.KEEP_DELETED_CELLS, HColumnDescriptor.REPLICATION_SCOPE); @@ -725,13 +725,13 @@ public class MetaDataUtil { } public static boolean propertyNotAllowedToBeOutOfSync(String colFamProp) { - return SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.contains(colFamProp); + return SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(colFamProp); } public static Map<String, Object> getSyncedProps(HColumnDescriptor defaultCFDesc) { Map<String, Object> syncedProps = new HashMap<>(); if (defaultCFDesc != null) { - for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { syncedProps.put(propToKeepInSync, Bytes.toString( defaultCFDesc.getValue(Bytes.toBytes(propToKeepInSync)))); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index ede14b8..8d0aca1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -46,6 +46,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT; @@ -140,6 +141,14 @@ public class UpgradeUtil { + "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, BASE_COLUMN_COUNT) " + "VALUES (?, ?, ?, ?, ?, ?) "; + public static final String UPSERT_UPDATE_CACHE_FREQUENCY = + "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + UPDATE_CACHE_FREQUENCY + + ") VALUES (?, ?, ?, ?)"; + public static String SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW = "SELECT " + "BASE_COLUMN_COUNT " + "FROM \"SYSTEM\".CATALOG " @@ -1227,7 +1236,7 @@ public class UpgradeUtil { for (HColumnDescriptor currentColFam: tableDesc.getColumnFamilies()) { if (!currentColFam.equals(defaultColFam)) { HColumnDescriptor newColFamDesc = new HColumnDescriptor(currentColFam); - for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop))); String expectedPropVal = syncedProps.get(prop).toString(); if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) { @@ -1299,6 +1308,71 @@ public class UpgradeUtil { } } + private static void syncUpdateCacheFreqForIndexesOfTable(PTable baseTable, + PreparedStatement stmt) throws SQLException { + for (PTable index : baseTable.getIndexes()) { + if (index.getUpdateCacheFrequency() == baseTable.getUpdateCacheFrequency()) { + continue; + } + stmt.setString(2, index.getSchemaName().getString()); + stmt.setString(3, index.getTableName().getString()); + stmt.setLong(4, baseTable.getUpdateCacheFrequency()); + stmt.addBatch(); + } + } + + /** + * See PHOENIX-4891. We set the UPDATE_CACHE_FREQUENCY of indexes to be same as their parent. + * We do this for both physical base tables as well as views + * @param conn Phoenix Connection object + * @param table PTable corresponding to a physical base table + * @throws SQLException + * @throws IOException + */ + public static void syncUpdateCacheFreqAllIndexes(PhoenixConnection conn, PTable table) + throws SQLException, IOException { + // Use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG + try(PhoenixConnection newConn = new PhoenixConnection(conn, HConstants.LATEST_TIMESTAMP)) { + // Clear the server-side cache so that we get the latest built PTables + newConn.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + byte[] tenantId = newConn.getTenantId() != null ? + newConn.getTenantId().getBytes() : null; + + PreparedStatement stmt = + newConn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY); + stmt.setString(1, Bytes.toString(tenantId)); + syncUpdateCacheFreqForIndexesOfTable(table, stmt); + + TableViewFinderResult childViewsResult = new TableViewFinderResult(); + try (Table childLinkTable = newConn.getQueryServices() + .getTable(SchemaUtil.getPhysicalName( + PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES, + newConn.getQueryServices().getProps()) + .getName())) { + ViewFinder.findAllRelatives(childLinkTable, tenantId, + table.getSchemaName().getBytes(), table.getTableName().getBytes(), + LinkType.CHILD_TABLE, childViewsResult); + + // Iterate over the chain of child views + for (TableInfo tableInfo: childViewsResult.getLinks()) { + PTable view; + String viewName = SchemaUtil.getTableName(tableInfo.getSchemaName(), + tableInfo.getTableName()); + try { + view = PhoenixRuntime.getTable(newConn, viewName); + } catch (TableNotFoundException e) { + // Ignore + logger.warn("Error getting PTable for view: " + viewName); + continue; + } + syncUpdateCacheFreqForIndexesOfTable(view, stmt); + } + } + stmt.executeBatch(); + newConn.commit(); + } + } + /** * Make sure that all tables have necessary column family properties in sync * with each other and also in sync with all the table's indexes @@ -1316,7 +1390,7 @@ public class UpgradeUtil { // Ignore physical view index tables since we handle them for each base table already continue; } - PTable table = null; + PTable table; String tableName = origTableDesc.getTableName().getNameAsString(); try { table = PhoenixRuntime.getTable(conn, tableName); @@ -1329,6 +1403,7 @@ public class UpgradeUtil { // Ignore global index tables since we handle them for each base table already continue; } + syncUpdateCacheFreqAllIndexes(conn, table); HColumnDescriptor defaultColFam = origTableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table)); Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(defaultColFam);
