http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java index f7edea7..416c452 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java @@ -24,7 +24,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -35,9 +34,8 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -47,12 +45,6 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator; -import org.apache.hadoop.hbase.regionserver.LocalIndexSplitter; -import org.apache.hadoop.hbase.regionserver.Store; -import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; @@ -63,19 +55,16 @@ import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTableKey; -import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; -import org.apache.phoenix.util.StringUtil; import org.apache.phoenix.util.TestUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -88,9 +77,6 @@ import com.google.common.collect.Maps; @RunWith(Parameterized.class) public class LocalIndexIT extends BaseHBaseManagedTimeIT { - private static CountDownLatch latch1 = new CountDownLatch(1); - private static CountDownLatch latch2 = new CountDownLatch(1); - private static final int WAIT_TIME_SECONDS = 60; private String schemaName="TEST"; private boolean isNamespaceMapped; private String tableName = schemaName + ".T"; @@ -102,8 +88,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { public LocalIndexIT(boolean isNamespaceMapped) { this.isNamespaceMapped = isNamespaceMapped; this.physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), isNamespaceMapped); - this.indexPhysicalTableName = Bytes - .toString(MetaDataUtil.getLocalIndexPhysicalName(physicalTableName.getName())); + this.indexPhysicalTableName = this.physicalTableName.getNameAsString(); } @BeforeClass @@ -150,6 +135,15 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { } @Test + public void testCreationOfTableWithLocalIndexColumnFamilyPrefixShouldFail() throws Exception { + Connection conn1 = DriverManager.getConnection(getUrl()); + try { + conn1.createStatement().execute("CREATE TABLE T(L#a varchar primary key, aL# integer)"); + fail("Column families specified in the table creation should not have local colunm prefix."); + } catch (SQLException e) { } + } + + @Test public void testLocalIndexCreationWithSplitsShouldFail() throws Exception { createBaseTable(tableName, null, null); Connection conn1 = getConnection(); @@ -215,11 +209,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { Connection conn2 = getConnection(); conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)"); conn2.createStatement().executeQuery("SELECT * FROM " + tableName).next(); - HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - assertTrue("Local index table should be present.", admin.tableExists(indexPhysicalTableName)); conn1.createStatement().execute("DROP TABLE "+ tableName); - admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - assertFalse("Local index table should be deleted.", admin.tableExists(indexPhysicalTableName)); ResultSet rs = conn2.createStatement().executeQuery("SELECT " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + "," + PhoenixDatabaseMetaData.SEQUENCE_NAME @@ -247,6 +237,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { byte[][] endKeys = startEndKeys.getSecond(); for (int i = 0; i < startKeys.length; i++) { Scan s = new Scan(); + s.addFamily(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES); s.setStartRow(startKeys[i]); s.setStopRow(endKeys[i]); ResultScanner scanner = indexTable.getScanner(s); @@ -280,6 +271,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { byte[][] endKeys = startEndKeys.getSecond(); for (int i = 0; i < startKeys.length; i++) { Scan s = new Scan(); + s.addFamily(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES); s.setStartRow(startKeys[i]); s.setStopRow(endKeys[i]); ResultScanner scanner = indexTable.getScanner(s); @@ -318,7 +310,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName + " [-32768,'a'] - [-32768,'b']\n" + + indexPhysicalTableName + " [1,'a'] - [1,'b']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -342,7 +334,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName + " [-32768,'a']\n" + + indexPhysicalTableName + " [1,'a']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -361,7 +353,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { rs = conn1.createStatement().executeQuery("EXPLAIN "+ query); assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + indexPhysicalTableName - + " [-32768,*] - [-32768,'z']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + + " [1,*] - [1,'z']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [\"K3\"]\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn1.createStatement().executeQuery(query); @@ -384,7 +376,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName +" [-32768]\n" + + indexPhysicalTableName +" [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -448,7 +440,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName + " [-32768,'a']\n" + + indexPhysicalTableName + " [1,'a']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -471,7 +463,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName +" [-32768,*] - [-32768,'z']\n" + + indexPhysicalTableName +" [1,*] - [1,'z']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -507,7 +499,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName +" [-32768,*] - [-32768,'z']\n" + + indexPhysicalTableName +" [1,*] - [1,'z']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -535,7 +527,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { rs = conn1.createStatement().executeQuery("EXPLAIN " + query); assertEquals( "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName +" [-32768,*] - [-32768,'z']\n" + + indexPhysicalTableName +" [1,*] - [1,'z']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -591,8 +583,8 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)"); conn1.createStatement().execute("DROP INDEX " + indexName + " ON " + tableName); HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - HTable indexTable = new HTable(admin.getConfiguration() ,TableName.valueOf(MetaDataUtil.getLocalIndexTableName(tableName))); - Pair<byte[][], byte[][]> startEndKeys = indexTable.getStartEndKeys(); + HTable table = new HTable(admin.getConfiguration() ,TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME)); + Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys(); byte[][] startKeys = startEndKeys.getFirst(); byte[][] endKeys = startEndKeys.getSecond(); // No entry should be present in local index table after drop index. @@ -600,7 +592,13 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { Scan s = new Scan(); s.setStartRow(startKeys[i]); s.setStopRow(endKeys[i]); - ResultScanner scanner = indexTable.getScanner(s); + Collection<HColumnDescriptor> families = table.getTableDescriptor().getFamilies(); + for(HColumnDescriptor cf: families) { + if(cf.getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)){ + s.addFamily(cf.getName()); + } + } + ResultScanner scanner = table.getScanner(s); int count = 0; for(Result r:scanner){ count++; @@ -608,7 +606,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { scanner.close(); assertEquals(0, count); } - indexTable.close(); + table.close(); } finally { conn1.close(); } @@ -720,7 +718,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("j", rs.getString("t_id")); - assertTrue(rs.next()); + assertTrue(rs.next()); assertEquals("b", rs.getString("t_id")); assertFalse(rs.next()); } finally { @@ -778,18 +776,6 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { admin.getConnection(), physicalTableName, false); } assertEquals(4+i, regionsOfUserTable.size()); - TableName indexTable = - TableName.valueOf(indexPhysicalTableName); - List<HRegionInfo> regionsOfIndexTable = - MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), indexTable, false); - - while (regionsOfIndexTable.size() != (4 + i)) { - Thread.sleep(100); - regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), indexTable, false); - } - assertEquals(4 + i, regionsOfIndexTable.size()); String[] tIdColumnValues = new String[26]; String[] v1ColumnValues = new String[26]; int[] k1ColumnValue = new int[26]; @@ -814,7 +800,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { rs = conn1.createStatement().executeQuery("EXPLAIN " + query); assertEquals( "CLIENT PARALLEL " + (4 + i) + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName + " [-32768]\n" + + indexPhysicalTableName + " [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -824,7 +810,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { "CLIENT PARALLEL " + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1)) + "-WAY RANGE SCAN OVER " - + indexPhysicalTableName + " [-32767]\n" + + indexPhysicalTableName + " [2]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn1.createStatement().executeQuery(query); @@ -927,19 +913,6 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(), physicalTableName, false); } - assertEquals(3, regionsOfUserTable.size()); - TableName indexTable = - TableName.valueOf(indexPhysicalTableName); - List<HRegionInfo> regionsOfIndexTable = - MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), indexTable, false); - - while (regionsOfIndexTable.size() != 3) { - Thread.sleep(100); - regionsOfIndexTable = MetaTableAccessor.getTableRegions( - getUtility().getZooKeeperWatcher(), admin.getConnection(), indexTable, false); - } - assertEquals(3, regionsOfIndexTable.size()); String query = "SELECT t_id,k1,v1 FROM " + tableName; rs = conn1.createStatement().executeQuery(query); Thread.sleep(1000); @@ -953,7 +926,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName - + " [-32768]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + + " [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); query = "SELECT t_id,k1,k3 FROM " + tableName; @@ -961,7 +934,7 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { assertEquals( "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName - + " [-32767]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + + " [2]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn1.createStatement().executeQuery(query); @@ -976,93 +949,4 @@ public class LocalIndexIT extends BaseHBaseManagedTimeIT { conn1.close(); } } - - @Test - public void testLocalIndexStateWhenSplittingInProgress() throws Exception { - if (isNamespaceMapped) { return; } - createBaseTable(tableName+"2", null, "('e','j','o')"); - Connection conn1 = getConnection(); - try{ - String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; - for (int i = 0; i < 26; i++) { - conn1.createStatement().execute( - "UPSERT INTO " + tableName+"2" + " values('"+strings[i]+"'," + i + "," - + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')"); - } - conn1.commit(); - conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName+"2" + "(v1)"); - conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName+"2" + "(k3)"); - - ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName+"2"); - assertTrue(rs.next()); - HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - HTableDescriptor tableDesc = admin.getTableDescriptor(SchemaUtil.getPhysicalTableName((tableName+"2").getBytes(),isNamespaceMapped)); - tableDesc.removeCoprocessor(LocalIndexSplitter.class.getName()); - tableDesc.addCoprocessor(MockedLocalIndexSplitter.class.getName(), null, - 1, null); - admin.disableTable(tableDesc.getTableName()); - admin.modifyTable(tableDesc.getTableName(), tableDesc); - admin.enableTable(tableDesc.getTableName()); - TableName indexTable = TableName.valueOf(MetaDataUtil.getLocalIndexTableName( - SchemaUtil.getPhysicalTableName((tableName + "2").getBytes(), isNamespaceMapped).getNameAsString())); - HTableDescriptor indexTableDesc = admin.getTableDescriptor(indexTable); - indexTableDesc.removeCoprocessor(IndexHalfStoreFileReaderGenerator.class.getName()); - indexTableDesc.addCoprocessor(MockedIndexHalfStoreFileReaderGenerator.class.getName(), null, - 1, null); - admin.disableTable(indexTable); - admin.modifyTable(indexTable, indexTableDesc); - admin.enableTable(indexTable); - - admin.split(SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName + "2"), isNamespaceMapped), - ByteUtil.concat(Bytes.toBytes(strings[3]))); - List<HRegionInfo> regionsOfUserTable = - admin.getTableRegions(SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName + "2"), isNamespaceMapped)); - - while (regionsOfUserTable.size() != 5) { - Thread.sleep(100); - regionsOfUserTable = admin.getTableRegions(SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName + "2"), isNamespaceMapped)); - } - assertEquals(5, regionsOfUserTable.size()); - - List<HRegionInfo> regionsOfIndexTable = admin.getTableRegions(indexTable); - - while (regionsOfIndexTable.size() != 5) { - Thread.sleep(100); - regionsOfIndexTable = admin.getTableRegions(indexTable); - } - - assertEquals(5, regionsOfIndexTable.size()); - boolean success = latch1.await(WAIT_TIME_SECONDS, TimeUnit.SECONDS); - assertTrue("Timed out waiting for MockedLocalIndexSplitter.preSplitAfterPONR to complete", success); - latch2.countDown(); - } finally { - conn1.close(); - latch1.countDown(); - latch2.countDown(); - } - } - - public static class MockedIndexHalfStoreFileReaderGenerator extends IndexHalfStoreFileReaderGenerator { - @Override - public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, - StoreFile resultFile) throws IOException { - try { - boolean success = latch2.await(WAIT_TIME_SECONDS, TimeUnit.SECONDS); - assertTrue("Timed out waiting for test to complete", success); - super.postCompact(e, store, resultFile); - } catch (InterruptedException e1) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e1); - } - } - } - - public static class MockedLocalIndexSplitter extends LocalIndexSplitter { - @Override - public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx) - throws IOException { - super.preSplitAfterPONR(ctx); - latch1.countDown(); - } - } }
http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index 7818bcc..5d0230b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Properties; +import java.util.Set; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; @@ -44,6 +45,7 @@ import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseOwnClusterHBaseManagedTimeIT; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTableType; @@ -210,6 +212,9 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT { conn.commit(); fail(); } catch (SQLException e) { + System.out.println(); + } catch(Exception e) { + System.out.println(); } // Verify the metadata for index is correct. @@ -292,8 +297,8 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT { rs = conn.createStatement().executeQuery("EXPLAIN " + query); expectedPlan = " OVER " + (localIndex - ? Bytes.toString(MetaDataUtil.getLocalIndexPhysicalName(SchemaUtil - .getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).getName())) + ? Bytes.toString(SchemaUtil + .getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).getName()) : SchemaUtil.getPhysicalTableName(fullIndexName.getBytes(), isNamespaceMapped).getNameAsString()); String explainPlan = QueryUtil.getExplainPlan(rs); assertTrue(explainPlan.contains(expectedPlan)); @@ -336,6 +341,13 @@ public class MutableIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT { if (c.getEnvironment().getRegionInfo().getTable().getNameAsString().contains(INDEX_NAME) && FAIL_WRITE) { throw new DoNotRetryIOException(); } + Mutation operation = miniBatchOp.getOperation(0); + Set<byte[]> keySet = operation.getFamilyMap().keySet(); + for(byte[] family: keySet) { + if(Bytes.toString(family).startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX) && FAIL_WRITE) { + throw new DoNotRetryIOException(); + } + } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index 80f1250..70bfdd5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@ -50,7 +50,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -117,7 +116,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { String query = "SELECT char_col1, int_col1, long_col2 from " + fullTableName; ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName +" [-32768]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName +" [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } else { assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, QueryUtil.getExplainPlan(rs)); } @@ -184,7 +183,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { if(localIndex) { query = "SELECT b.* from " + fullTableName + " where int_col1 = 4"; rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName +" [-32768]\n" + + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName +" [1]\n" + " SERVER FILTER BY TO_INTEGER(\"INT_COL1\") = 4\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -247,7 +246,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { query = "SELECT * FROM " + fullTableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if(localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName+" [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } else { assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, QueryUtil.getExplainPlan(rs)); } @@ -268,7 +267,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { query = "SELECT * FROM " + fullTableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if(localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName + " [-32768]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName + " [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } else { assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, QueryUtil.getExplainPlan(rs)); } @@ -289,7 +288,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { query = "SELECT * FROM " + fullTableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if(localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName+" [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } else { assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, QueryUtil.getExplainPlan(rs)); } @@ -354,7 +353,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { query = "SELECT * FROM " + fullTableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\n" + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName+" [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } else { @@ -478,7 +477,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { query = "SELECT * FROM " + fullTableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if(localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + fullTableName+" [-32768]\n" + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullTableName+" [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); @@ -620,28 +619,38 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { Properties props = new Properties(); props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(2)); props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false)); - try(Connection conn1 = DriverManager.getConnection(getUrl(), props)){ + Connection conn1 = DriverManager.getConnection(getUrl()); + HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); + dropTable(admin, conn1); + try{ String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; - HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - dropTable(admin, conn1); createTableAndLoadData(conn1, strings, isReverse); ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName); assertTrue(rs.next()); splitDuringScan(conn1, strings, admin, isReverse); dropTable(admin, conn1); - } + } finally { + dropTable(admin, conn1); + if(conn1 != null) conn1.close(); + if(admin != null) admin.close(); + } } private void dropTable(HBaseAdmin admin, Connection conn) throws SQLException, IOException { - conn.createStatement().execute("DROP TABLE IF EXISTS "+ tableName); - if(admin.tableExists(tableName)) { - admin.disableTable(TableName.valueOf(tableName)); - admin.deleteTable(TableName.valueOf(tableName)); - } - if(admin.tableExists(localIndex? MetaDataUtil.getLocalIndexTableName(tableName): indexName)) { - admin.disableTable(localIndex? MetaDataUtil.getLocalIndexTableName(tableName): indexName); - admin.deleteTable(localIndex? MetaDataUtil.getLocalIndexTableName(tableName): indexName); + try { + conn.createStatement().execute("DROP TABLE IF EXISTS "+ tableName); + } finally { + if(admin.tableExists(tableName)) { + admin.disableTable(TableName.valueOf(tableName)); + admin.deleteTable(TableName.valueOf(tableName)); + } + if(!localIndex) { + if(admin.tableExists(indexName)) { + admin.disableTable(TableName.valueOf(indexName)); + admin.deleteTable(TableName.valueOf(indexName)); + } + } } } @@ -677,7 +686,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { assertEquals(4, rs.getInt(1)); TableName table = TableName.valueOf(localIndex?tableName: indexName); - TableName indexTable = TableName.valueOf(localIndex?MetaDataUtil.getLocalIndexTableName(tableName): indexName); + TableName indexTable = TableName.valueOf(localIndex?tableName: indexName); admin.flush(indexTable); boolean merged = false; // merge regions until 1 left @@ -697,7 +706,7 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { if(!merged) { List<HRegionInfo> regions = admin.getTableRegions(localIndex ? table : indexTable); - System.out.println("Merging: " + regions.size()); + Log.info("Merging: " + regions.size()); admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), regions.get(1).getEncodedNameAsBytes(), false); merged = true; @@ -707,16 +716,17 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { } catch (Exception ex) { Log.info(ex); } - - long waitStartTime = System.currentTimeMillis(); - // wait until merge happened - while (System.currentTimeMillis() - waitStartTime < 10000) { - List<HRegionInfo> regions = admin.getTableRegions(indexTable); - System.out.println("Waiting:" + regions.size()); - if (regions.size() < numRegions) { - break; - } - Threads.sleep(1000); + if(!localIndex) { + long waitStartTime = System.currentTimeMillis(); + // wait until merge happened + while (System.currentTimeMillis() - waitStartTime < 10000) { + List<HRegionInfo> regions = admin.getTableRegions(indexTable); + Log.info("Waiting:" + regions.size()); + if (regions.size() < numRegions) { + break; + } + Threads.sleep(1000); + } } } } @@ -725,36 +735,6 @@ public class MutableIndexIT extends BaseHBaseManagedTimeIT { } } - private List<HRegionInfo> mergeRegions(HBaseAdmin admin, List<HRegionInfo> regionsOfUserTable) - throws IOException, InterruptedException { - for (int i = 2; i > 0; i--) { - Threads.sleep(10000); - admin.mergeRegions(regionsOfUserTable.get(0).getEncodedNameAsBytes(), - regionsOfUserTable.get(1).getEncodedNameAsBytes(), false); - regionsOfUserTable = - MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(), - TableName.valueOf(localIndex? tableName:indexName), false); - - while (regionsOfUserTable.size() != i) { - Thread.sleep(100); - regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), TableName.valueOf(localIndex? tableName:indexName), false); - } - assertEquals(i, regionsOfUserTable.size()); - if(localIndex) { - List<HRegionInfo> regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), TableName.valueOf(MetaDataUtil.getLocalIndexTableName(tableName)), false); - while (regionsOfIndexTable.size() != i) { - Thread.sleep(100); - regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), - admin.getConnection(), TableName.valueOf(MetaDataUtil.getLocalIndexTableName(tableName)), false); - } - assertEquals(i, regionsOfIndexTable.size()); - } - } - return regionsOfUserTable; - } - private List<HRegionInfo> splitDuringScan(Connection conn1, String[] strings, HBaseAdmin admin, boolean isReverse) throws SQLException, IOException, InterruptedException { ResultSet rs; http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java index 931fcae..a3ad418 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Properties; +import java.util.Set; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; @@ -41,11 +42,13 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseOwnClusterHBaseManagedTimeIT; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.hbase.index.Indexer; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTableType; @@ -279,6 +282,13 @@ public class ReadOnlyIndexFailureIT extends BaseOwnClusterHBaseManagedTimeIT { if (c.getEnvironment().getRegionInfo().getTable().getNameAsString().contains(INDEX_NAME) && FAIL_WRITE) { throw new DoNotRetryIOException(); } + Mutation operation = miniBatchOp.getOperation(0); + Set<byte[]> keySet = operation.getFamilyMap().keySet(); + for(byte[] family: keySet) { + if(Bytes.toString(family).startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX) && FAIL_WRITE) { + throw new DoNotRetryIOException(); + } + } } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java index 450cffa..6b980a4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java @@ -113,7 +113,7 @@ public class ViewIndexIT extends BaseHBaseManagedTimeIT { public ViewIndexIT(boolean isNamespaceMapped) { this.isNamespaceMapped = isNamespaceMapped; this.physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), isNamespaceMapped); - this.viewIndexPhysicalTableName = MetaDataUtil.getLocalIndexTableName(physicalTableName.getNameAsString()); + this.viewIndexPhysicalTableName = this.physicalTableName.getNameAsString(); } @@ -172,7 +172,7 @@ public class ViewIndexIT extends BaseHBaseManagedTimeIT { String sql = "SELECT * FROM " + VIEW_NAME + " WHERE v2 = 100"; ResultSet rs = conn1.prepareStatement("EXPLAIN " + sql).executeQuery(); assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_T ['10',-32768,100]\n" + + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['10',1,100]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn1.prepareStatement(sql).executeQuery(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java index e675dda..558bdcb 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java @@ -33,8 +33,11 @@ import java.util.Collection; import java.util.Map; import java.util.Properties; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -305,7 +308,11 @@ public class MutableRollbackIT extends BaseHBaseManagedTimeIT { conn.rollback(); assertDataAndIndexRows(stmt); - + PhoenixConnection phoenixConn = conn.unwrap(PhoenixConnection.class); + if(localIndex) { + dropTable(phoenixConn.getQueryServices().getAdmin(), conn, fullTableName1); + dropTable(phoenixConn.getQueryServices().getAdmin(), conn, fullTableName2); + } } finally { conn.close(); } @@ -440,7 +447,8 @@ public class MutableRollbackIT extends BaseHBaseManagedTimeIT { assertEquals("x", rs.getString(1)); assertEquals("yyyy", rs.getString(2)); assertFalse(rs.next()); - + PhoenixConnection phoenixConn = conn.unwrap(PhoenixConnection.class); + if(localIndex) dropTable(phoenixConn.getQueryServices().getAdmin(), conn, fullTableName1); } finally { conn.close(); } @@ -503,9 +511,19 @@ public class MutableRollbackIT extends BaseHBaseManagedTimeIT { assertEquals("x", rs.getString(1)); assertEquals("a", rs.getString(2)); assertFalse(rs.next()); - + PhoenixConnection phoenixConn = conn.unwrap(PhoenixConnection.class); + if(localIndex) dropTable(phoenixConn.getQueryServices().getAdmin(), conn, fullTableName1); } finally { conn.close(); } } + + private void dropTable(HBaseAdmin admin, Connection conn, String tableName) throws SQLException, IOException { + conn.createStatement().execute("DROP TABLE IF EXISTS "+ tableName); + if(admin.tableExists(tableName)) { + admin.disableTable(TableName.valueOf(tableName)); + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java index 3e1d5ab..f3fb46a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java @@ -76,7 +76,7 @@ public class RollbackIT extends BaseHBaseManagedTimeIT { public static Collection<Boolean[]> data() { return Arrays.asList(new Boolean[][] { { false, false }, { false, true }, - { true, false }, { true, true } + { true, false }, { true, true } }); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java index 8e36bf0..c7364dd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java @@ -152,7 +152,7 @@ public class TxWriteFailureIT extends BaseOwnClusterHBaseManagedTimeIT { rs = conn.createStatement().executeQuery("EXPLAIN "+indexSql); if(localIndex) { assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_" + DATA_TABLE_FULL_NAME + " [-32768]\n" + + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + DATA_TABLE_FULL_NAME + " [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java deleted file mode 100644 index 1bec821..0000000 --- a/phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java +++ /dev/null @@ -1,485 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.hbase.index.balancer; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; -import org.apache.hadoop.hbase.master.RegionStates; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.TestSplitTransactionOnCluster.MockedRegionObserver; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; -import org.apache.phoenix.hbase.index.IndexTestingUtils; -import org.apache.phoenix.hbase.index.Indexer; -import org.apache.phoenix.hbase.index.master.IndexMasterObserver; -import org.apache.phoenix.util.ConfigUtil; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.Timeout; - -@Category(NeedsOwnMiniClusterTest.class) -public class IndexLoadBalancerIT { - - private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - private static HBaseAdmin admin = null; - - @Rule - public Timeout timeout = new Timeout(300, TimeUnit.SECONDS); - - @BeforeClass - public static void setupCluster() throws Exception { - final int NUM_RS = 4; - Configuration conf = UTIL.getConfiguration(); - conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true); - conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); - conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class, - LoadBalancer.class); - IndexTestingUtils.setupConfig(conf); - // disable version checking, so we can test against whatever version of HBase happens to be - // installed (right now, its generally going to be SNAPSHOT versions). - conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false); - // set replication required parameter - ConfigUtil.setReplicationConfigIfAbsent(conf); - UTIL.startMiniCluster(NUM_RS); - admin = UTIL.getHBaseAdmin(); - } - - @Test - public void testRoundRobinAssignmentDuringIndexTableCreation() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - TableName tableName = TableName.valueOf("testRoundRobinAssignmentDuringIndexTableCreation"); - TableName indexTableName = - TableName.valueOf("testRoundRobinAssignmentDuringIndexTableCreation_index"); - createUserAndIndexTable(tableName, indexTableName); - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - } - - @Test - public void testColocationAfterSplit() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - // Table names to make use of the - TableName tableName = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_1"); - TableName indexTableName = TableName.valueOf("testSplitHooksBeforeAndAfterPONR_2"); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addCoprocessor(MockedRegionObserver.class.getName()); - htd.addFamily(new HColumnDescriptor("cf")); - char c = 'A'; - byte[][] split = new byte[20][]; - for (int i = 0; i < 20; i++) { - byte[] b = { (byte) c }; - split[i] = b; - c++; - } - admin.createTable(htd, split); - HTableDescriptor iHtd = new HTableDescriptor(indexTableName); - iHtd.addFamily(new HColumnDescriptor("cf")); - iHtd.setValue(IndexLoadBalancer.PARENT_TABLE_KEY, tableName.toBytes()); - admin.createTable(iHtd, split); - - // test put with the indexed column - - insertData(tableName); - insertData(indexTableName); - - admin.split(tableName.getNameAsString(), "c"); - List<HRegionInfo> regionsOfUserTable = - master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName); - - while (regionsOfUserTable.size() != 22) { - Thread.sleep(100); - regionsOfUserTable = - master.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName); - } - - List<HRegionInfo> regionsOfIndexTable = - master.getAssignmentManager().getRegionStates().getRegionsOfTable(indexTableName); - - while (regionsOfIndexTable.size() != 22) { - Thread.sleep(100); - regionsOfIndexTable = - master.getAssignmentManager().getRegionStates().getRegionsOfTable( - indexTableName); - } - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - } - - @Test - public void testColocationAfterRegionsMerge() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - RegionStates regionStates = master.getAssignmentManager().getRegionStates(); - // Table names to make use of the - TableName tableName = TableName.valueOf("testColocationAfterRegionsMerge"); - TableName indexTableName = TableName.valueOf("testColocationAfterRegionsMerge_index"); - createUserAndIndexTable(tableName, indexTableName); - ServerName server = cluster.getRegionServer(0).getServerName(); - List<HRegionInfo> regionsOfUserTable = regionStates.getRegionsOfTable(tableName); - Pair<HRegionInfo, HRegionInfo> regionsToMerge = new Pair<HRegionInfo, HRegionInfo>(); - byte[] startKey1 = { (byte) 'C' }; - byte[] startKey2 = { (byte) 'D' }; - for (HRegionInfo region : regionsOfUserTable) { - if (Bytes.compareTo(startKey1, region.getStartKey()) == 0) { - regionsToMerge.setFirst(region); - } else if (Bytes.compareTo(startKey2, region.getStartKey()) == 0) { - regionsToMerge.setSecond(region); - } - } - admin.move(regionsToMerge.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(server - .toString())); - admin.move(regionsToMerge.getSecond().getEncodedNameAsBytes(), Bytes.toBytes(server - .toString())); - - List<HRegionInfo> regionsOfIndexTable = regionStates.getRegionsOfTable(indexTableName); - Pair<HRegionInfo, HRegionInfo> indexRegionsToMerge = new Pair<HRegionInfo, HRegionInfo>(); - for (HRegionInfo region : regionsOfIndexTable) { - if (Bytes.compareTo(startKey1, region.getStartKey()) == 0) { - indexRegionsToMerge.setFirst(region); - } else if (Bytes.compareTo(startKey2, region.getStartKey()) == 0) { - indexRegionsToMerge.setSecond(region); - } - } - admin.move(indexRegionsToMerge.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(server - .toString())); - admin.move(indexRegionsToMerge.getSecond().getEncodedNameAsBytes(), Bytes.toBytes(server - .toString())); - while (!regionStates.getRegionServerOfRegion(regionsToMerge.getFirst()).equals(server) - || !regionStates.getRegionServerOfRegion(regionsToMerge.getSecond()).equals(server) - || !regionStates.getRegionServerOfRegion(indexRegionsToMerge.getFirst()).equals( - server) - || !regionStates.getRegionServerOfRegion(indexRegionsToMerge.getSecond()).equals( - server)) { - Threads.sleep(1000); - } - admin.mergeRegions(regionsToMerge.getFirst().getEncodedNameAsBytes(), regionsToMerge - .getSecond().getEncodedNameAsBytes(), true); - admin.mergeRegions(indexRegionsToMerge.getFirst().getEncodedNameAsBytes(), - indexRegionsToMerge.getSecond().getEncodedNameAsBytes(), true); - - while (regionsOfUserTable.size() != 20 || regionsOfIndexTable.size() != 20) { - Thread.sleep(100); - regionsOfUserTable = regionStates.getRegionsOfTable(tableName); - regionsOfIndexTable = regionStates.getRegionsOfTable(indexTableName); - } - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - } - - private void insertData(TableName tableName) throws IOException, InterruptedException { - HTable table = new HTable(admin.getConfiguration(), tableName); - Put p = new Put("a".getBytes()); - p.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); - p.add("cf".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); - table.put(p); - - Put p1 = new Put("b".getBytes()); - p1.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); - p1.add("cf".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); - table.put(p1); - - Put p2 = new Put("c".getBytes()); - p2.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); - p2.add("cf".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); - table.put(p2); - - Put p3 = new Put("c1".getBytes()); - p3.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); - p3.add("cf".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); - table.put(p3); - - Put p4 = new Put("d".getBytes()); - p4.add(Bytes.toBytes("cf"), Bytes.toBytes("q1"), Bytes.toBytes("Val")); - p4.add("cf".getBytes(), "q2".getBytes(), Bytes.toBytes("ValForCF2")); - table.put(p4); - admin.flush(tableName.getNameAsString()); - } - - @Test - public void testRandomAssignmentDuringIndexTableEnable() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); - TableName tableName = TableName.valueOf("testRandomAssignmentDuringIndexTableEnable"); - TableName indexTableName = - TableName.valueOf("testRandomAssignmentDuringIndexTableEnable_index"); - createUserAndIndexTable(tableName, indexTableName); - admin.disableTable(tableName); - admin.disableTable(indexTableName); - admin.enableTable(tableName); - admin.enableTable(indexTableName); - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - - } - - @Test - public void testBalanceCluster() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - master.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", false); - master.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); - master.getConfiguration().setBoolean("hbase.master.loadbalance.bytable", false); - - TableName tableName = TableName.valueOf("testBalanceCluster"); - TableName indexTableName = TableName.valueOf("testBalanceCluster_index"); - createUserAndIndexTable(tableName, indexTableName); - HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf("testBalanceCluster1")); - htd1.addFamily(new HColumnDescriptor("fam1")); - char c = 'A'; - byte[][] split1 = new byte[12][]; - for (int i = 0; i < 12; i++) { - byte[] b = { (byte) c }; - split1[i] = b; - c++; - } - admin.setBalancerRunning(false, false); - admin.createTable(htd1, split1); - admin.disableTable(tableName); - admin.enableTable(tableName); - admin.setBalancerRunning(true, false); - admin.balancer(); - boolean isRegionsColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionsColocated); - } - - @Test - public void testBalanceByTable() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - master.getConfiguration().setBoolean("hbase.master.loadbalance.bytable", true); - TableName tableName = TableName.valueOf("testBalanceByTable"); - TableName indexTableName = TableName.valueOf("testBalanceByTable_index"); - createUserAndIndexTable(tableName, indexTableName); - HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf("testBalanceByTable1")); - htd1.addFamily(new HColumnDescriptor("fam1")); - char c = 'A'; - byte[][] split1 = new byte[12][]; - for (int i = 0; i < 12; i++) { - byte[] b = { (byte) c }; - split1[i] = b; - c++; - } - admin.disableTable(tableName); - admin.enableTable(tableName); - admin.setBalancerRunning(true, false); - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - admin.balancer(); - Thread.sleep(10000); - ZKAssign.blockUntilNoRIT(zkw); - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Threads.sleep(1000); - } - isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - } - - @Test - public void testRoundRobinAssignmentAfterRegionServerDown() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - TableName tableName = TableName.valueOf("testRoundRobinAssignmentAfterRegionServerDown"); - TableName indexTableName = - TableName.valueOf("testRoundRobinAssignmentAfterRegionServerDown_index"); - createUserAndIndexTable(tableName, indexTableName); - HRegionServer regionServer = cluster.getRegionServer(1); - regionServer.abort("Aborting to test random assignment after region server down"); - while (master.getServerManager().areDeadServersInProgress()) { - Thread.sleep(1000); - } - ZKAssign.blockUntilNoRIT(zkw); - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Threads.sleep(1000); - } - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - - } - - @Test - public void testRetainAssignmentDuringMasterStartUp() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - master.getConfiguration().setBoolean("hbase.master.startup.retainassign", true); - TableName tableName = TableName.valueOf("testRetainAssignmentDuringMasterStartUp"); - TableName indexTableName = - TableName.valueOf("testRetainAssignmentDuringMasterStartUp_index"); - createUserAndIndexTable(tableName, indexTableName); - UTIL.shutdownMiniHBaseCluster(); - UTIL.startMiniHBaseCluster(1, 4); - cluster = UTIL.getHBaseCluster(); - master = cluster.getMaster(); - if (admin != null) { - admin.close(); - admin = new HBaseAdmin(master.getConfiguration()); - } - ZKAssign.blockUntilNoRIT(zkw); - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Threads.sleep(1000); - } - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - - } - - @Ignore // FIXME: PHOENIX-2625 - @Test - public void testRoundRobinAssignmentDuringMasterStartUp() throws Exception { - MiniHBaseCluster cluster = UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - UTIL.getConfiguration().setBoolean("hbase.master.startup.retainassign", false); - - TableName tableName = TableName.valueOf("testRoundRobinAssignmentDuringMasterStartUp"); - TableName indexTableName = - TableName.valueOf("testRoundRobinAssignmentDuringMasterStartUp_index"); - createUserAndIndexTable(tableName, indexTableName); - UTIL.shutdownMiniHBaseCluster(); - cluster.waitUntilShutDown(); - UTIL.startMiniHBaseCluster(1, 4); - cluster = UTIL.getHBaseCluster(); - if (admin != null) { - admin.close(); - admin = new HBaseAdmin(cluster.getMaster().getConfiguration()); - } - master = cluster.getMaster(); - while (master.getAssignmentManager().getRegionStates().isRegionsInTransition()) { - Threads.sleep(1000); - } - boolean isRegionColocated = - checkForColocation(master, tableName.getNameAsString(), indexTableName - .getNameAsString()); - assertTrue("User regions and index regions should colocate.", isRegionColocated); - } - - private void createUserAndIndexTable(TableName tableName, TableName indexTableName) - throws IOException { - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor("cf")); - char c = 'A'; - byte[][] split = new byte[20][]; - for (int i = 0; i < 20; i++) { - byte[] b = { (byte) c }; - split[i] = b; - c++; - } - admin.createTable(htd, split); - HTableDescriptor iHtd = new HTableDescriptor(indexTableName); - iHtd.addFamily(new HColumnDescriptor("cf")); - iHtd.setValue(IndexLoadBalancer.PARENT_TABLE_KEY, tableName.toBytes()); - admin.createTable(iHtd, split); - } - - private List<Pair<byte[], ServerName>> getStartKeysAndLocations(HMaster master, String tableName) - throws IOException, InterruptedException { - - List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations = - MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), - TableName.valueOf(tableName)); - List<Pair<byte[], ServerName>> startKeyAndLocationPairs = - new ArrayList<Pair<byte[], ServerName>>(tableRegionsAndLocations.size()); - Pair<byte[], ServerName> startKeyAndLocation = null; - for (Pair<HRegionInfo, ServerName> regionAndLocation : tableRegionsAndLocations) { - startKeyAndLocation = - new Pair<byte[], ServerName>(regionAndLocation.getFirst().getStartKey(), - regionAndLocation.getSecond()); - startKeyAndLocationPairs.add(startKeyAndLocation); - } - return startKeyAndLocationPairs; - - } - - public boolean checkForColocation(HMaster master, String tableName, String indexTableName) - throws IOException, InterruptedException { - List<Pair<byte[], ServerName>> uTableStartKeysAndLocations = - getStartKeysAndLocations(master, tableName); - List<Pair<byte[], ServerName>> iTableStartKeysAndLocations = - getStartKeysAndLocations(master, indexTableName); - - boolean regionsColocated = true; - if (uTableStartKeysAndLocations.size() != iTableStartKeysAndLocations.size()) { - regionsColocated = false; - } else { - for (int i = 0; i < uTableStartKeysAndLocations.size(); i++) { - Pair<byte[], ServerName> uStartKeyAndLocation = uTableStartKeysAndLocations.get(i); - Pair<byte[], ServerName> iStartKeyAndLocation = iTableStartKeysAndLocations.get(i); - - if (Bytes.compareTo(uStartKeyAndLocation.getFirst(), iStartKeyAndLocation - .getFirst()) == 0) { - if (uStartKeyAndLocation.getSecond().equals(iStartKeyAndLocation.getSecond())) { - continue; - } - } - regionsColocated = false; - } - } - return regionsColocated; - } -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/10909ae5/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java index 6cf8fa1..7b5287a 100644 --- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java +++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; @@ -74,6 +75,9 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver { HRegionInfo childRegion = region.getRegionInfo(); byte[] splitKey = null; if (reader == null && r != null) { + if(!p.toString().contains(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)) { + return super.preStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, reader); + } Scan scan = MetaTableAccessor.getScanForTableName(tableName); SingleColumnValueFilter scvf = null; if (Reference.isTopFileRegion(r.getFileRegion())) { @@ -140,8 +144,7 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver { try { conn = QueryUtil.getConnectionOnServer(ctx.getEnvironment().getConfiguration()).unwrap( PhoenixConnection.class); - String userTableName = MetaDataUtil.getUserTableName(tableName.getNameAsString()); - PTable dataTable = PhoenixRuntime.getTable(conn, userTableName); + PTable dataTable = PhoenixRuntime.getTable(conn, tableName.getNameAsString()); List<PTable> indexes = dataTable.getIndexes(); Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers = new HashMap<ImmutableBytesWritable, IndexMaintainer>(); @@ -180,7 +183,11 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver { public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s, CompactionRequest request) throws IOException { - if (!scanType.equals(ScanType.COMPACT_DROP_DELETES) || s != null || !store.hasReferences()) { + if (!store.getFamily().getNameAsString() + .startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX) + || !scanType.equals(ScanType.COMPACT_DROP_DELETES) + || s != null + || !store.hasReferences()) { return s; } List<StoreFileScanner> newScanners = new ArrayList<StoreFileScanner>(scanners.size()); @@ -236,7 +243,9 @@ public class IndexHalfStoreFileReaderGenerator extends BaseRegionObserver { public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store, final Scan scan, final NavigableSet<byte[]> targetCols, final KeyValueScanner s) throws IOException { - if(store.hasReferences()) { + if (store.getFamily().getNameAsString() + .startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX) + && store.hasReferences()) { long readPt = c.getEnvironment().getRegion().getReadpoint(scan.getIsolationLevel()); boolean scanUsePread = c.getEnvironment().getConfiguration().getBoolean("hbase.storescanner.use.pread", scan.isSmall()); Collection<StoreFile> storeFiles = store.getStorefiles();
