Repository: phoenix Updated Branches: refs/heads/master 91f085a90 -> 4e8c9ab0c
PHOENIX-4008 UPDATE STATISTIC should collect all versions of cells Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4e8c9ab0 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4e8c9ab0 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4e8c9ab0 Branch: refs/heads/master Commit: 4e8c9ab0c081dbad7db50f717b1c1ef1afd79a80 Parents: 91f085a Author: Bin Shi <[email protected]> Authored: Tue Sep 18 12:19:51 2018 -0700 Committer: Karan Mehta <[email protected]> Committed: Thu Sep 20 23:06:00 2018 -0700 ---------------------------------------------------------------------- .../phoenix/schema/stats/StatsCollectorIT.java | 72 ++++++++++++++------ .../apache/phoenix/schema/MetaDataClient.java | 1 + 2 files changed, 53 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e8c9ab0/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java index 3af0d09..41e39e3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver; @@ -698,6 +697,25 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { assertEquals("Number of expected rows in stats table after major compaction didn't match", numRows, rs.getInt(1)); } } + + private void verifyGuidePostGenerated(ConnectionQueryServices queryServices, + String tableName, String[] familyNames, + long guidePostWidth, boolean emptyGuidePostExpected) throws Exception { + try (Table statsHTable = + queryServices.getTable( + SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, + queryServices.getProps()).getName())) { + for (String familyName : familyNames) { + GuidePostsInfo gps = + StatisticsUtil.readStatistics(statsHTable, + new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes(familyName)), + HConstants.LATEST_TIMESTAMP); + assertTrue(emptyGuidePostExpected ? gps.isEmptyGuidePost() : !gps.isEmptyGuidePost()); + assertTrue(gps.getByteCounts()[0] >= guidePostWidth); + assertTrue(gps.getGuidePostTimestamps()[0] > 0); + } + } + } @Test public void testEmptyGuidePostGeneratedWhenDataSizeLessThanGPWidth() throws Exception { @@ -714,25 +732,39 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { conn.createStatement().execute("UPDATE STATISTICS " + tableName); ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices(); - try (Table statsHTable = - queryServices.getTable( - SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, - queryServices.getProps()).getName())) { - GuidePostsInfo gps = - StatisticsUtil.readStatistics(statsHTable, - new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C1")), - HConstants.LATEST_TIMESTAMP); - assertTrue(gps.isEmptyGuidePost()); - assertEquals(guidePostWidth, gps.getByteCounts()[0]); - assertTrue(gps.getGuidePostTimestamps()[0] > 0); - gps = - StatisticsUtil.readStatistics(statsHTable, - new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C2")), - HConstants.LATEST_TIMESTAMP); - assertTrue(gps.isEmptyGuidePost()); - assertEquals(guidePostWidth, gps.getByteCounts()[0]); - assertTrue(gps.getGuidePostTimestamps()[0] > 0); - } + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true); + } + } + + @Test + public void testCollectingAllVersionsOfCells() throws Exception { + String tableName = generateUniqueName(); + try (Connection conn = DriverManager.getConnection(getUrl())) { + long guidePostWidth = 70; + String ddl = + "CREATE TABLE " + tableName + " (k INTEGER PRIMARY KEY, c1.a bigint, c2.b bigint)" + + " GUIDE_POSTS_WIDTH=" + guidePostWidth + + ", USE_STATS_FOR_PARALLELIZATION=true" + ", VERSIONS=3"; + conn.createStatement().execute(ddl); + conn.createStatement().execute("upsert into " + tableName + " values (100,100,3)"); + conn.commit(); + conn.createStatement().execute("UPDATE STATISTICS " + tableName); + + ConnectionQueryServices queryServices = + conn.unwrap(PhoenixConnection.class).getQueryServices(); + + // The table only has one row. All cells just has one version, and the data size of the row + // is less than the guide post width, so we generate empty guide post. + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true); + + + conn.createStatement().execute("upsert into " + tableName + " values (100,101,4)"); + conn.commit(); + conn.createStatement().execute("UPDATE STATISTICS " + tableName); + + // We updated the row. Now each cell has two versions, and the data size of the row + // is >= the guide post width, so we generate non-empty guide post. + verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, false); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e8c9ab0/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index cdbf234..5c79603 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1279,6 +1279,7 @@ public class MetaDataClient { MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp); Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); + scan.readAllVersions(); scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES); boolean runUpdateStatsAsync = props.getBoolean(QueryServices.RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); scan.setAttribute(RUN_UPDATE_STATS_ASYNC_ATTRIB, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES);
