This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new 82241d4 PHOENIX-5962 Stabilize builds 82241d4 is described below commit 82241d4e152023d87bf4cd400942d1c0c2df6562 Author: Richard Antal <antal97rich...@gmail.com> AuthorDate: Tue Jun 16 16:20:39 2020 +0200 PHOENIX-5962 Stabilize builds --- .../phoenix/end2end/IndexBuildTimestampIT.java | 144 +++++++++++---------- .../phoenix/end2end/IndexToolTimeRangeIT.java | 33 ++--- .../end2end/IndexVerificationOldDesignIT.java | 2 + .../NonParameterizedIndexScrutinyToolIT.java | 4 + .../index/IndexVerificationOutputRepositoryIT.java | 2 + .../phoenix/util/EnvironmentEdgeManager.java | 3 + .../phoenix/index/VerifySingleIndexRowTest.java | 22 ++-- .../pherf/workload/MultiThreadedRunnerTest.java | 38 +++--- 8 files changed, 139 insertions(+), 109 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java index 591efd5..2441af4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java @@ -170,80 +170,84 @@ public class IndexBuildTimestampIT extends BaseUniqueNamesOwnClusterIT { @Test public void testCellTimestamp() throws Exception { EnvironmentEdgeManager.reset(); - MyClock clock1 = new MyClock(100000); - MyClock clock2 = new MyClock(200000); - String dataTableName = generateUniqueName(); - populateTable(dataTableName, clock1, clock2); - - MyClock clock3 = new MyClock(300000); - EnvironmentEdgeManager.injectEdge(clock3); + try { + MyClock clock1 = new MyClock(100000); + MyClock clock2 = new MyClock(200000); + String dataTableName = generateUniqueName(); + populateTable(dataTableName, clock1, clock2); + + MyClock clock3 = new MyClock(300000); + EnvironmentEdgeManager.injectEdge(clock3); + + Properties props = new Properties(); + props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS, "true"); + props.setProperty(QueryServices.ENABLE_SERVER_SIDE_DELETE_MUTATIONS, "true"); + Connection conn = DriverManager.getConnection(getUrl(), props); + + String viewName = null; + if (view) { + viewName = generateUniqueName(); + conn.createStatement().execute("CREATE VIEW "+ viewName + " AS SELECT * FROM " + + dataTableName); + } + String indexName = generateUniqueName(); + conn.createStatement().execute("CREATE "+ (localIndex ? "LOCAL " : "") + " INDEX " + indexName + " on " + + (view ? viewName : dataTableName) + " (val) include (ts)" + (async ? "ASYNC" : "")); - Properties props = new Properties(); - props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS, "true"); - props.setProperty(QueryServices.ENABLE_SERVER_SIDE_DELETE_MUTATIONS, "true"); - Connection conn = DriverManager.getConnection(getUrl(), props); - - String viewName = null; - if (view) { - viewName = generateUniqueName(); - conn.createStatement().execute("CREATE VIEW "+ viewName + " AS SELECT * FROM " + - dataTableName); - } - String indexName = generateUniqueName(); - conn.createStatement().execute("CREATE "+ (localIndex ? "LOCAL " : "") + " INDEX " + indexName + " on " + - (view ? viewName : dataTableName) + " (val) include (ts)" + (async ? "ASYNC" : "")); + conn.close(); - conn.close(); + if (async) { + // run the index MR job. + IndexToolIT.runIndexTool(true, false, null, (view ? viewName : dataTableName), indexName); + } - if (async) { - // run the index MR job. - IndexToolIT.runIndexTool(true, false, null, (view ? viewName : dataTableName), indexName); + // Verify the index timestamps via Phoenix + String selectSql = String.format("SELECT * FROM %s WHERE val = 'abc'", (view ? viewName : dataTableName)); + conn = DriverManager.getConnection(getUrl()); + // assert we are pulling from index table + assertExplainPlan(conn, localIndex, selectSql, dataTableName, (view ? "_IDX_" + dataTableName : indexName)); + ResultSet rs = conn.createStatement().executeQuery(selectSql); + assertTrue (rs.next()); + assertTrue(rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() < clock2.initialTime() && + rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() >= clock1.initialTime()); + + selectSql = + String.format("SELECT * FROM %s WHERE val = 'bcd'", (view ? viewName : dataTableName)); + // assert we are pulling from index table + assertExplainPlan(conn, localIndex, selectSql, dataTableName, (view ? "_IDX_" + dataTableName : indexName)); + + rs = conn.createStatement().executeQuery(selectSql); + assertTrue (rs.next()); + assertTrue(rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() < clock3.initialTime() && + rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() >= clock2.initialTime() + ); + assertFalse (rs.next()); + + // Verify the index timestamps via HBase + PTable pIndexTable = PhoenixRuntime.getTable(conn, indexName); + Table table = conn.unwrap(PhoenixConnection.class).getQueryServices() + .getTable(pIndexTable.getPhysicalName().getBytes()); + + Scan scan = new Scan(); + scan.setTimeRange(clock3.initialTime(), clock3.currentTime()); + ResultScanner scanner = table.getScanner(scan); + assertTrue(scanner.next() == null); + + + scan = new Scan(); + scan.setTimeRange(clock2.initialTime(), clock3.initialTime()); + scanner = table.getScanner(scan); + assertTrue(scanner.next() != null); + + + scan = new Scan(); + scan.setTimeRange(clock1.initialTime(), clock2.initialTime()); + scanner = table.getScanner(scan); + assertTrue(scanner.next() != null); + conn.close(); + } finally { + EnvironmentEdgeManager.reset(); } - // Verify the index timestamps via Phoenix - String selectSql = String.format("SELECT * FROM %s WHERE val = 'abc'", (view ? viewName : dataTableName)); - conn = DriverManager.getConnection(getUrl()); - // assert we are pulling from index table - assertExplainPlan(conn, localIndex, selectSql, dataTableName, (view ? "_IDX_" + dataTableName : indexName)); - ResultSet rs = conn.createStatement().executeQuery(selectSql); - assertTrue (rs.next()); - assertTrue(rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() < clock2.initialTime() && - rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() >= clock1.initialTime()); - - selectSql = - String.format("SELECT * FROM %s WHERE val = 'bcd'", (view ? viewName : dataTableName)); - // assert we are pulling from index table - assertExplainPlan(conn, localIndex, selectSql, dataTableName, (view ? "_IDX_" + dataTableName : indexName)); - - rs = conn.createStatement().executeQuery(selectSql); - assertTrue (rs.next()); - assertTrue(rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() < clock3.initialTime() && - rs.unwrap(PhoenixResultSet.class).getCurrentRow().getValue(0).getTimestamp() >= clock2.initialTime() - ); - assertFalse (rs.next()); - - // Verify the index timestamps via HBase - PTable pIndexTable = PhoenixRuntime.getTable(conn, indexName); - Table table = conn.unwrap(PhoenixConnection.class).getQueryServices() - .getTable(pIndexTable.getPhysicalName().getBytes()); - - Scan scan = new Scan(); - scan.setTimeRange(clock3.initialTime(), clock3.currentTime()); - ResultScanner scanner = table.getScanner(scan); - assertTrue(scanner.next() == null); - - - scan = new Scan(); - scan.setTimeRange(clock2.initialTime(), clock3.initialTime()); - scanner = table.getScanner(scan); - assertTrue(scanner.next() != null); - - - scan = new Scan(); - scan.setTimeRange(clock1.initialTime(), clock2.initialTime()); - scanner = table.getScanner(scan); - assertTrue(scanner.next() != null); - conn.close(); - EnvironmentEdgeManager.reset(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java index d188fe3..04a3b83 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java @@ -76,23 +76,26 @@ public class IndexToolTimeRangeIT extends BaseUniqueNamesOwnClusterIT { private static void populateDataTable() throws SQLException { myClock = new MyClock(); EnvironmentEdgeManager.injectEdge(myClock); - //So that we don't have to recompute the values below - myClock.tickTime(); - myClock.tickTime(); - try (Connection conn = getConnection()) { - //row 1-> time 4, row 2-> time 5, row 3-> time 6, row 4-> time 7, row 5-> time 8 - for (int i=0; i<5; i++) { - myClock.tickTime(); - PreparedStatement ps = conn.prepareStatement( - String.format(UPSERT_TABLE_DML, dataTableFullName)); - ps.setInt(1, i+1); - ps.setInt(2,(i+1)*10); - ps.setInt(3, (i+1)*100); - ps.execute(); - conn.commit(); + try { + //So that we don't have to recompute the values below + myClock.tickTime(); + myClock.tickTime(); + try (Connection conn = getConnection()) { + //row 1-> time 4, row 2-> time 5, row 3-> time 6, row 4-> time 7, row 5-> time 8 + for (int i=0; i<5; i++) { + myClock.tickTime(); + PreparedStatement ps = conn.prepareStatement( + String.format(UPSERT_TABLE_DML, dataTableFullName)); + ps.setInt(1, i+1); + ps.setInt(2,(i+1)*10); + ps.setInt(3, (i+1)*100); + ps.execute(); + conn.commit(); + } } + } finally { + EnvironmentEdgeManager.reset(); } - EnvironmentEdgeManager.injectEdge(null); } private static void setupMiniCluster() throws Exception { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java index 477f74f..a045161 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java @@ -110,6 +110,8 @@ public class IndexVerificationOldDesignIT extends BaseUniqueNamesOwnClusterIT { assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue()); + } finally { + EnvironmentEdgeManager.reset(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java index 93a723d..8b36518 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java @@ -178,6 +178,8 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT assertEquals(2, getCounterValue(counters, EXPIRED_ROW_COUNT)); assertEquals(0, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT)); + } finally { + EnvironmentEdgeManager.reset(); } } @@ -220,6 +222,8 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT assertEquals(2, getCounterValue(counters, EXPIRED_ROW_COUNT)); assertEquals(0, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT)); + } finally { + EnvironmentEdgeManager.reset(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java index 61fc6f2..96386df 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java @@ -135,6 +135,8 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT int count = TestUtil.getRowCount(hTable, false); Assert.assertEquals(0, count); + } finally { + EnvironmentEdgeManager.reset(); } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EnvironmentEdgeManager.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EnvironmentEdgeManager.java index 775dd8c..c5e1a63 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/EnvironmentEdgeManager.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EnvironmentEdgeManager.java @@ -52,6 +52,9 @@ public class EnvironmentEdgeManager { * Injects the given edge such that it becomes the managed entity. If null is * passed to this method, the default type is assigned to the delegate. * + * <b>Note: This is JVM global. Make sure to call reset() after the test.</b> + * See org.apache.hadoop.hbase.util.EnvironmentEdgeManager for other caveats + * * @param edge the new edge. */ public static void injectEdge(EnvironmentEdge edge) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java index 6ad9da7..1797e48 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java @@ -358,15 +358,19 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest { @Test public void testVerifySingleIndexRow_expiredIndexRowCount_nonZero() throws IOException { IndexToolVerificationResult.PhaseResult - expectedPR = new IndexToolVerificationResult.PhaseResult(0, 1, 0, 0, 0,0); - for (Map.Entry<byte[], List<Mutation>> - entry : indexKeyToMutationMapLocal.entrySet()) { - initializeLocalMockitoSetup(entry, TestType.EXPIRED); - expireThisRow(); - //test code - rebuildScanner.verifySingleIndexRow(indexRow, actualPR); - - assertTrue(actualPR.equals(expectedPR)); + expectedPR = new IndexToolVerificationResult.PhaseResult(0, 1, 0, 0, 0, 0); + try { + for (Map.Entry<byte[], List<Mutation>> + entry : indexKeyToMutationMapLocal.entrySet()) { + initializeLocalMockitoSetup(entry, TestType.EXPIRED); + expireThisRow(); + //test code + rebuildScanner.verifySingleIndexRow(indexRow, actualPR); + + assertTrue(actualPR.equals(expectedPR)); + } + } finally { + EnvironmentEdgeManager.reset(); } } @Ignore diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/MultiThreadedRunnerTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/MultiThreadedRunnerTest.java index 592aceb..66f65b3 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/MultiThreadedRunnerTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/MultiThreadedRunnerTest.java @@ -93,10 +93,14 @@ public class MultiThreadedRunnerTest { DefaultEnvironmentEdge myClock = Mockito.mock(DefaultEnvironmentEdge.class); Mockito.when(myClock.currentTime()).thenReturn(0L, 5000L); EnvironmentEdgeManager.injectEdge(myClock); - Mockito.when(mockRS.next()).thenReturn(true); - Mockito.when(mockRS.getLong(1)).thenReturn(1L); - Pair<Long, Long> results = mtr.getResults(mockRS, "test_iteration", false,0L); - assertTrue(results.getSecond()>mockQuery.getTimeoutDuration()); + try { + Mockito.when(mockRS.next()).thenReturn(true); + Mockito.when(mockRS.getLong(1)).thenReturn(1L); + Pair<Long, Long> results = mtr.getResults(mockRS, "test_iteration", false, 0L); + assertTrue(results.getSecond() > mockQuery.getTimeoutDuration()); + } finally { + EnvironmentEdgeManager.reset(); + } } @Test @@ -104,17 +108,21 @@ public class MultiThreadedRunnerTest { DefaultEnvironmentEdge myClock = Mockito.mock(DefaultEnvironmentEdge.class); Mockito.when(myClock.currentTime()).thenReturn(0L); EnvironmentEdgeManager.injectEdge(myClock); - Mockito.when(mockQuery.getTimeoutDuration()).thenReturn(1000L); - Mockito.when(mockQuery.getExpectedAggregateRowCount()).thenReturn(1L); - MultiThreadedRunner mtr = new MultiThreadedRunner("test", - mockQuery, mockDMR, mockTT, - 10L, 1000L, - true, mockRA, - mockScenario, mockWE, mockParser); - Mockito.when(mockRS.next()).thenReturn(true, false); - Mockito.when(mockRS.getLong(1)).thenReturn(1L); - Pair<Long, Long> results = mtr.getResults(mockRS, "test_iteration", false, 0L); - assertFalse(results.getSecond() > mockQuery.getTimeoutDuration()); + try { + Mockito.when(mockQuery.getTimeoutDuration()).thenReturn(1000L); + Mockito.when(mockQuery.getExpectedAggregateRowCount()).thenReturn(1L); + MultiThreadedRunner mtr = new MultiThreadedRunner("test", + mockQuery, mockDMR, mockTT, + 10L, 1000L, + true, mockRA, + mockScenario, mockWE, mockParser); + Mockito.when(mockRS.next()).thenReturn(true, false); + Mockito.when(mockRS.getLong(1)).thenReturn(1L); + Pair<Long, Long> results = mtr.getResults(mockRS, "test_iteration", false, 0L); + assertFalse(results.getSecond() > mockQuery.getTimeoutDuration()); + } finally { + EnvironmentEdgeManager.reset(); + } } }