Repository: phoenix Updated Branches: refs/heads/master 442d8eb29 -> d2575288d
PHOENIX-3855 Separate MutableIndexIT into multiple test classes Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d2575288 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d2575288 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d2575288 Branch: refs/heads/master Commit: d2575288d1542c5b6e8dbe65448a22cf59aca8ff Parents: 442d8eb Author: James Taylor <[email protected]> Authored: Tue May 16 16:09:57 2017 -0700 Committer: James Taylor <[email protected]> Committed: Tue May 16 16:09:57 2017 -0700 ---------------------------------------------------------------------- .../phoenix/end2end/index/MutableIndexIT.java | 162 ++++++----------- .../index/MutableIndexSplitForwardScanIT.java | 33 ++++ .../end2end/index/MutableIndexSplitIT.java | 180 +++++++++++++++++++ .../index/MutableIndexSplitReverseScanIT.java | 33 ++++ 4 files changed, 297 insertions(+), 111 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2575288/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index 424099d..e46a213 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@ -42,9 +42,6 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; @@ -60,7 +57,6 @@ import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -626,57 +622,6 @@ public class MutableIndexIT extends ParallelStatsDisabledIT { } } - @Test - public void testSplitDuringIndexScan() throws Exception { - testSplitDuringIndexScan(false); - } - - @Test - public void testSplitDuringIndexReverseScan() throws Exception { - testSplitDuringIndexScan(true); - } - - private void testSplitDuringIndexScan(boolean isReverse) throws Exception { - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(2)); - props.setProperty(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false)); - Connection conn1 = getConnection(props); - String tableName = "TBL_" + generateUniqueName(); - String indexName = "IDX_" + generateUniqueName(); - HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - dropTable(admin, conn1); - try{ - String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; - createTableAndLoadData(conn1, tableName, indexName, strings, isReverse); - - ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName); - assertTrue(rs.next()); - splitDuringScan(conn1, tableName, indexName, strings, admin, isReverse); - dropTable(admin, conn1); - - } finally { - dropTable(admin, conn1); - if(conn1 != null) conn1.close(); - if(admin != null) admin.close(); - } - } - - private void dropTable(HBaseAdmin admin, Connection conn) throws SQLException, IOException { - - String tableName = "TBL_" + generateUniqueName(); - String indexName = "IDX_" + generateUniqueName(); - conn.createStatement().execute("DROP TABLE IF EXISTS "+ tableName); - if(admin.tableExists(tableName)) { - admin.disableTable(TableName.valueOf(tableName)); - admin.deleteTable(TableName.valueOf(tableName)); - } - if(!localIndex && admin.tableExists(indexName)) { - admin.disableTable(indexName); - admin.deleteTable(indexName); - - } - } - private void createTableAndLoadData(Connection conn1, String tableName, String indexName, String[] strings, boolean isReverse) throws SQLException { createBaseTable(conn1, tableName, null); for (int i = 0; i < 26; i++) { @@ -696,65 +641,60 @@ public class MutableIndexIT extends ParallelStatsDisabledIT { HBaseAdmin admin = connectionQueryServices.getAdmin(); String tableName = "TBL_" + generateUniqueName(); String indexName = "IDX_" + generateUniqueName(); - try { - dropTable(admin, conn1); - createBaseTable(conn1, tableName, "('e')"); - conn1.createStatement().execute("CREATE "+(localIndex?"LOCAL":"")+" INDEX " + indexName + " ON " + tableName + "(v1)" + (localIndex?"":" SPLIT ON ('e')")); - conn1.createStatement().execute("UPSERT INTO "+tableName+" values('b',1,2,4,'z')"); - conn1.createStatement().execute("UPSERT INTO "+tableName+" values('f',1,2,3,'z')"); - conn1.createStatement().execute("UPSERT INTO "+tableName+" values('j',2,4,2,'a')"); - conn1.createStatement().execute("UPSERT INTO "+tableName+" values('q',3,1,1,'c')"); - conn1.commit(); - + createBaseTable(conn1, tableName, "('e')"); + conn1.createStatement().execute("CREATE "+(localIndex?"LOCAL":"")+" INDEX " + indexName + " ON " + tableName + "(v1)" + (localIndex?"":" SPLIT ON ('e')")); + conn1.createStatement().execute("UPSERT INTO "+tableName+" values('b',1,2,4,'z')"); + conn1.createStatement().execute("UPSERT INTO "+tableName+" values('f',1,2,3,'z')"); + conn1.createStatement().execute("UPSERT INTO "+tableName+" values('j',2,4,2,'a')"); + conn1.createStatement().execute("UPSERT INTO "+tableName+" values('q',3,1,1,'c')"); + conn1.commit(); + - String query = "SELECT count(*) FROM " + tableName +" where v1<='z'"; - ResultSet rs = conn1.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals(4, rs.getInt(1)); + String query = "SELECT count(*) FROM " + tableName +" where v1<='z'"; + ResultSet rs = conn1.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals(4, rs.getInt(1)); - TableName indexTable = TableName.valueOf(localIndex?tableName: indexName); - admin.flush(indexTable); - boolean merged = false; - HTableInterface table = connectionQueryServices.getTable(indexTable.getName()); - // merge regions until 1 left - long numRegions = 0; - while (true) { - rs = conn1.createStatement().executeQuery(query); - assertTrue(rs.next()); - assertEquals(4, rs.getInt(1)); //TODO this returns 5 sometimes instead of 4, duplicate results? - try { - List<HRegionInfo> indexRegions = admin.getTableRegions(indexTable); - numRegions = indexRegions.size(); - if (numRegions==1) { - break; - } - if(!merged) { - List<HRegionInfo> regions = - admin.getTableRegions(indexTable); - Log.info("Merging: " + regions.size()); - admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), - regions.get(1).getEncodedNameAsBytes(), false); - merged = true; - Threads.sleep(10000); - } - } catch (Exception ex) { - Log.info(ex); - } - long waitStartTime = System.currentTimeMillis(); - // wait until merge happened - while (System.currentTimeMillis() - waitStartTime < 10000) { - List<HRegionInfo> regions = admin.getTableRegions(indexTable); - Log.info("Waiting:" + regions.size()); - if (regions.size() < numRegions) { - break; - } - Threads.sleep(1000); - } - SnapshotTestingUtils.waitForTableToBeOnline(BaseTest.getUtility(), indexTable); - assertTrue("Index table should be online ", admin.isTableAvailable(indexTable)); + TableName indexTable = TableName.valueOf(localIndex?tableName: indexName); + admin.flush(indexTable); + boolean merged = false; + HTableInterface table = connectionQueryServices.getTable(indexTable.getName()); + // merge regions until 1 left + long numRegions = 0; + while (true) { + rs = conn1.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals(4, rs.getInt(1)); //TODO this returns 5 sometimes instead of 4, duplicate results? + try { + List<HRegionInfo> indexRegions = admin.getTableRegions(indexTable); + numRegions = indexRegions.size(); + if (numRegions==1) { + break; + } + if(!merged) { + List<HRegionInfo> regions = + admin.getTableRegions(indexTable); + Log.info("Merging: " + regions.size()); + admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(), + regions.get(1).getEncodedNameAsBytes(), false); + merged = true; + Threads.sleep(10000); } - } finally { - dropTable(admin, conn1); + } catch (Exception ex) { + Log.info(ex); + } + long waitStartTime = System.currentTimeMillis(); + // wait until merge happened + while (System.currentTimeMillis() - waitStartTime < 10000) { + List<HRegionInfo> regions = admin.getTableRegions(indexTable); + Log.info("Waiting:" + regions.size()); + if (regions.size() < numRegions) { + break; + } + Threads.sleep(1000); + } + SnapshotTestingUtils.waitForTableToBeOnline(BaseTest.getUtility(), indexTable); + assertTrue("Index table should be online ", admin.isTableAvailable(indexTable)); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2575288/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java new file mode 100644 index 0000000..4ce2b4c --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.index; + +import org.junit.Test; + +public class MutableIndexSplitForwardScanIT extends MutableIndexSplitIT { + + public MutableIndexSplitForwardScanIT(boolean localIndex) { + super(localIndex); + } + + @Test + public void testSplitDuringIndexScan() throws Exception { + testSplitDuringIndexScan(false); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2575288/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitIT.java new file mode 100644 index 0000000..77bd125 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitIT.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.index; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.TestUtil; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public abstract class MutableIndexSplitIT extends ParallelStatsDisabledIT { + + protected final boolean localIndex; + + public MutableIndexSplitIT(boolean localIndex) { + this.localIndex = localIndex; + } + + private static Connection getConnection(Properties props) throws SQLException { + props.setProperty(QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, Integer.toString(1)); + Connection conn = DriverManager.getConnection(getUrl(), props); + return conn; + } + + @Parameters(name="MutableIndexSplitIT_localIndex={0}") // name is used by failsafe as file name in reports + public static Collection<Boolean[]> data() { + return Arrays.asList(new Boolean[][] { + { false }, { true } }); + } + + protected void testSplitDuringIndexScan(boolean isReverse) throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(2)); + props.setProperty(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false)); + Connection conn1 = getConnection(props); + String tableName = "TBL_" + generateUniqueName(); + String indexName = "IDX_" + generateUniqueName(); + HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); + try{ + String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; + createTableAndLoadData(conn1, tableName, indexName, strings, isReverse); + + ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName); + assertTrue(rs.next()); + splitDuringScan(conn1, tableName, indexName, strings, admin, isReverse); + } finally { + if(conn1 != null) conn1.close(); + if(admin != null) admin.close(); + } + } + + private void createTableAndLoadData(Connection conn1, String tableName, String indexName, String[] strings, boolean isReverse) throws SQLException { + createBaseTable(conn1, tableName, null); + for (int i = 0; i < 26; i++) { + conn1.createStatement().execute( + "UPSERT INTO " + tableName + " values('"+strings[i]+"'," + i + "," + + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')"); + } + conn1.commit(); + conn1.createStatement().execute( + "CREATE " + (localIndex ? "LOCAL" : "")+" INDEX " + indexName + " ON " + tableName + "(v1"+(isReverse?" DESC":"")+") include (k3)"); + } + + private List<HRegionInfo> splitDuringScan(Connection conn1, String tableName, String indexName, String[] strings, HBaseAdmin admin, boolean isReverse) + throws SQLException, IOException, InterruptedException { + ResultSet rs; + + String query = "SELECT t_id,k1,v1 FROM " + tableName; + rs = conn1.createStatement().executeQuery(query); + String[] tIdColumnValues = new String[26]; + String[] v1ColumnValues = new String[26]; + int[] k1ColumnValue = new int[26]; + for (int j = 0; j < 5; j++) { + assertTrue(rs.next()); + tIdColumnValues[j] = rs.getString("t_id"); + k1ColumnValue[j] = rs.getInt("k1"); + v1ColumnValues[j] = rs.getString("V1"); + } + + String[] splitKeys = new String[2]; + splitKeys[0] = strings[4]; + splitKeys[1] = strings[12]; + + int[] splitInts = new int[2]; + splitInts[0] = 22; + splitInts[1] = 4; + List<HRegionInfo> regionsOfUserTable = null; + for(int i = 0; i <=1; i++) { + Threads.sleep(10000); + if(localIndex) { + admin.split(Bytes.toBytes(tableName), + ByteUtil.concat(Bytes.toBytes(splitKeys[i]))); + } else { + admin.split(Bytes.toBytes(indexName), ByteUtil.concat(Bytes.toBytes(splitInts[i]))); + } + Thread.sleep(100); + regionsOfUserTable = + MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), + admin.getConnection(), TableName.valueOf(localIndex?tableName:indexName), + false); + + while (regionsOfUserTable.size() != (i+2)) { + Thread.sleep(100); + regionsOfUserTable = + MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), + admin.getConnection(), + TableName.valueOf(localIndex?tableName:indexName), false); + } + assertEquals(i+2, regionsOfUserTable.size()); + } + for (int j = 5; j < 26; j++) { + assertTrue(rs.next()); + tIdColumnValues[j] = rs.getString("t_id"); + k1ColumnValue[j] = rs.getInt("k1"); + v1ColumnValues[j] = rs.getString("V1"); + } + Arrays.sort(tIdColumnValues); + Arrays.sort(v1ColumnValues); + Arrays.sort(k1ColumnValue); + assertTrue(Arrays.equals(strings, tIdColumnValues)); + assertTrue(Arrays.equals(strings, v1ColumnValues)); + for(int i=0;i<26;i++) { + assertEquals(i, k1ColumnValue[i]); + } + assertFalse(rs.next()); + return regionsOfUserTable; + } + + private void createBaseTable(Connection conn, String tableName, String splits) throws SQLException { + String ddl = "CREATE TABLE " + tableName + " (t_id VARCHAR NOT NULL,\n" + + "k1 INTEGER NOT NULL,\n" + + "k2 INTEGER NOT NULL,\n" + + "k3 INTEGER,\n" + + "v1 VARCHAR,\n" + + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2))\n" + + (splits != null ? (" split on " + splits) : ""); + conn.createStatement().execute(ddl); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2575288/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java new file mode 100644 index 0000000..dc9a3fa --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.index; + +import org.junit.Test; + +public class MutableIndexSplitReverseScanIT extends MutableIndexSplitIT { + + public MutableIndexSplitReverseScanIT(boolean localIndex) { + super(localIndex); + } + + @Test + public void testSplitDuringIndexScan() throws Exception { + testSplitDuringIndexScan(true); + } + +}
