Repository: phoenix Updated Branches: refs/heads/master d6a4caf5c -> 9fc37c7cd
PHOENIX-1511 Invalidate stats row even if not running stats due to phoenix.stats.minUpdateFrequency Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9fc37c7c Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9fc37c7c Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9fc37c7c Branch: refs/heads/master Commit: 9fc37c7cd0ce33337c5fd0b133bf43bd7024d6d7 Parents: d6a4caf Author: James Taylor <[email protected]> Authored: Tue Dec 30 10:51:48 2014 -0800 Committer: James Taylor <[email protected]> Committed: Tue Dec 30 10:51:48 2014 -0800 ---------------------------------------------------------------------- .../phoenix/end2end/StatsCollectorIT.java | 60 ++++++++++++++++---- .../apache/phoenix/schema/MetaDataClient.java | 33 ++++++----- 2 files changed, 66 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/9fc37c7c/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java index ca8ab81..8b3bdee 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java @@ -21,6 +21,7 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.getAllSplits; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -43,6 +44,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -228,28 +230,39 @@ public class StatsCollectorIT extends BaseOwnClusterHBaseManagedTimeIT { return stmt; } - private void compactTable(Connection conn) throws IOException, InterruptedException, SQLException { + private void compactTable(Connection conn, String tableName) throws IOException, InterruptedException, SQLException { ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices(); HBaseAdmin admin = services.getAdmin(); try { - admin.flush(STATS_TEST_TABLE_NAME); - admin.majorCompact(STATS_TEST_TABLE_NAME); + admin.flush(tableName); + admin.majorCompact(tableName); Thread.sleep(10000); // FIXME: how do we know when compaction is done? } finally { admin.close(); } - services.clearCache(); } @Test public void testCompactUpdatesStats() throws Exception { + testCompactUpdatesStats(null, STATS_TEST_TABLE_NAME + 1); + } + + @Test + public void testCompactUpdatesStatsWithMinStatsUpdateFreq() throws Exception { + testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, STATS_TEST_TABLE_NAME + 2); + } + + private void testCompactUpdatesStats(Integer minStatsUpdateFreq, String tableName) throws Exception { int nRows = 10; Connection conn; PreparedStatement stmt; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + if (minStatsUpdateFreq != null) { + props.setProperty(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, minStatsUpdateFreq.toString()); + } conn = DriverManager.getConnection(getUrl(), props); - conn.createStatement().execute("CREATE TABLE " + STATS_TEST_TABLE_NAME + "(k CHAR(1) PRIMARY KEY, v INTEGER) " + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE); - stmt = conn.prepareStatement("UPSERT INTO " + STATS_TEST_TABLE_NAME + " VALUES(?,?)"); + conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER) " + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE); + stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?)"); for (int i = 0; i < nRows; i++) { stmt.setString(1, Character.toString((char) ('a' + i))); stmt.setInt(2, i); @@ -257,18 +270,41 @@ public class StatsCollectorIT extends BaseOwnClusterHBaseManagedTimeIT { } conn.commit(); - compactTable(conn); - conn = DriverManager.getConnection(getUrl(), props); - List<KeyRange>keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME); + compactTable(conn, tableName); + if (minStatsUpdateFreq == null) { + conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + } + // Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run + // UPDATATE STATISTICS, the new statistics are faulted in as expected. + if (minStatsUpdateFreq != null) { + List<KeyRange>keyRanges = getAllSplits(conn, tableName); + assertNotEquals(nRows+1, keyRanges.size()); + // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache + // and forcing the new stats to be pulled over. + int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName); + assertEquals(0, rowCount); + } + List<KeyRange>keyRanges = getAllSplits(conn, tableName); assertEquals(nRows+1, keyRanges.size()); - int nDeletedRows = conn.createStatement().executeUpdate("DELETE FROM " + STATS_TEST_TABLE_NAME + " WHERE V < 5"); + int nDeletedRows = conn.createStatement().executeUpdate("DELETE FROM " + tableName + " WHERE V < 5"); conn.commit(); assertEquals(5, nDeletedRows); - compactTable(conn); + compactTable(conn, tableName); + if (minStatsUpdateFreq == null) { + conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache(); + } - keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME); + keyRanges = getAllSplits(conn, tableName); + if (minStatsUpdateFreq != null) { + assertEquals(nRows+1, keyRanges.size()); + // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache + // and force us to pull over the new stats + int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName); + assertEquals(0, rowCount); + keyRanges = getAllSplits(conn, tableName); + } assertEquals(nRows/2+1, keyRanges.size()); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/9fc37c7c/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 5b1315f..623fc0e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -658,25 +658,28 @@ public class MetaDataClient { if (rs.next()) { msSinceLastUpdate = rs.getLong(1) - rs.getLong(2); } - if (msSinceLastUpdate < msMinBetweenUpdates) { - return 0; + long rowCount = 0; + if (msSinceLastUpdate >= msMinBetweenUpdates) { + /* + * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through, + * since it may not represent a "real" table in the case of the view indexes of a base table. + */ + PostDDLCompiler compiler = new PostDDLCompiler(connection); + TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false); + MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp); + Scan scan = plan.getContext().getScan(); + scan.setCacheBlocks(false); + scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES); + MutationState mutationState = plan.execute(); + rowCount = mutationState.getUpdateCount(); } /* - * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through, - * since it may not represent a "real" table in the case of the view indexes of a base table. + * Update the stats table so that client will pull the new one with the updated stats. + * Even if we don't run the command due to the last update time, invalidate the cache. + * This supports scenarios in which a major compaction was manually initiated and the + * client wants the modified stats to be reflected immediately. */ - PostDDLCompiler compiler = new PostDDLCompiler(connection); - TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false); - MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp); - Scan scan = plan.getContext().getScan(); - scan.setCacheBlocks(false); - scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES); - MutationState mutationState = plan.execute(); - long rowCount = mutationState.getUpdateCount(); - - // We need to update the stats table so that client will pull the new one with - // the updated stats. connection.getQueryServices().clearTableFromCache(tenantIdBytes, Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())), Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
