Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.4 567220ae8 -> 3f22dc76f
PHOENIX-4008: UPDATE STATISTIC should collect all versions of cells. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f22dc76 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f22dc76 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f22dc76 Branch: refs/heads/4.x-HBase-1.4 Commit: 3f22dc76f2061531d98ed68cb050171144cf5ed8 Parents: 567220a Author: Bin <[email protected]> Authored: Mon Sep 24 12:31:19 2018 -0700 Committer: Karan Mehta <[email protected]> Committed: Mon Sep 24 23:27:51 2018 -0700 ---------------------------------------------------------------------- .../phoenix/schema/stats/StatsCollectorIT.java | 73 ++++++++++++++------ .../apache/phoenix/schema/MetaDataClient.java | 1 + 2 files changed, 54 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f22dc76/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java index 09d28f8..6eb9411 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java @@ -697,7 +697,26 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { assertEquals("Number of expected rows in stats table after major compaction didn't match", numRows, rs.getInt(1)); } } - + + private void verifyGuidePostGenerated(ConnectionQueryServices queryServices, + String tableName, String[] familyNames, + long guidePostWidth, boolean emptyGuidePostExpected) throws Exception { + try (HTableInterface statsHTable = + queryServices.getTable( + SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, + queryServices.getProps()).getName())) { + for (String familyName : familyNames) { + GuidePostsInfo gps = + StatisticsUtil.readStatistics(statsHTable, + new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes(familyName)), + HConstants.LATEST_TIMESTAMP); + assertTrue(emptyGuidePostExpected ? gps.isEmptyGuidePost() : !gps.isEmptyGuidePost()); + assertTrue(gps.getByteCounts()[0] >= guidePostWidth); + assertTrue(gps.getGuidePostTimestamps()[0] > 0); + } + } + } + @Test public void testEmptyGuidePostGeneratedWhenDataSizeLessThanGPWidth() throws Exception { String tableName = generateUniqueName(); @@ -713,25 +732,39 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { conn.createStatement().execute("UPDATE STATISTICS " + tableName); ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices(); - try (HTableInterface statsHTable = - queryServices.getTable( - SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, - queryServices.getProps()).getName())) { - GuidePostsInfo gps = - StatisticsUtil.readStatistics(statsHTable, - new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C1")), - HConstants.LATEST_TIMESTAMP); - assertTrue(gps.isEmptyGuidePost()); - assertEquals(guidePostWidth, gps.getByteCounts()[0]); - assertTrue(gps.getGuidePostTimestamps()[0] > 0); - gps = - StatisticsUtil.readStatistics(statsHTable, - new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C2")), - HConstants.LATEST_TIMESTAMP); - assertTrue(gps.isEmptyGuidePost()); - assertEquals(guidePostWidth, gps.getByteCounts()[0]); - assertTrue(gps.getGuidePostTimestamps()[0] > 0); - } + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true); + } + } + + @Test + public void testCollectingAllVersionsOfCells() throws Exception { + String tableName = generateUniqueName(); + try (Connection conn = DriverManager.getConnection(getUrl())) { + long guidePostWidth = 70; + String ddl = + "CREATE TABLE " + tableName + " (k INTEGER PRIMARY KEY, c1.a bigint, c2.b bigint)" + + " GUIDE_POSTS_WIDTH=" + guidePostWidth + + ", USE_STATS_FOR_PARALLELIZATION=true" + ", VERSIONS=3"; + conn.createStatement().execute(ddl); + conn.createStatement().execute("upsert into " + tableName + " values (100,100,3)"); + conn.commit(); + conn.createStatement().execute("UPDATE STATISTICS " + tableName); + + ConnectionQueryServices queryServices = + conn.unwrap(PhoenixConnection.class).getQueryServices(); + + // The table only has one row. All cells just has one version, and the data size of the row + // is less than the guide post width, so we generate empty guide post. + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true); + + + conn.createStatement().execute("upsert into " + tableName + " values (100,101,4)"); + conn.commit(); + conn.createStatement().execute("UPDATE STATISTICS " + tableName); + + // We updated the row. Now each cell has two versions, and the data size of the row + // is >= the guide post width, so we generate non-empty guide post. + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, false); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f22dc76/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index ecf3cd0..ac4cdad 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1283,6 +1283,7 @@ public class MetaDataClient { MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp); Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); + scan.setMaxVersions(); scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES); boolean runUpdateStatsAsync = props.getBoolean(QueryServices.RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); scan.setAttribute(RUN_UPDATE_STATS_ASYNC_ATTRIB, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES);
