PHOENIX-3361 Collect stats correct for local indexes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/eaf99f23 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/eaf99f23 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/eaf99f23 Branch: refs/heads/calcite Commit: eaf99f23da47b86e7577837c9c73ad47ad9ef0d1 Parents: 711d7f0 Author: James Taylor <[email protected]> Authored: Thu Oct 6 22:27:10 2016 -0700 Committer: James Taylor <[email protected]> Committed: Fri Oct 7 17:19:15 2016 -0700 ---------------------------------------------------------------------- .../apache/phoenix/end2end/GroupByCaseIT.java | 7 +- .../apache/phoenix/end2end/IndexExtendedIT.java | 7 +- .../phoenix/end2end/RoundFloorCeilFuncIT.java | 1 - .../phoenix/end2end/StatsCollectorIT.java | 53 +++-- .../end2end/index/MutableIndexFailureIT.java | 2 - .../phoenix/end2end/index/MutableIndexIT.java | 1 - .../phoenix/end2end/index/ViewIndexIT.java | 85 +++---- .../UngroupedAggregateRegionObserver.java | 8 +- .../phoenix/iterate/BaseResultIterators.java | 84 +++---- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 6 +- .../phoenix/query/ConnectionQueryServices.java | 20 +- .../query/ConnectionQueryServicesImpl.java | 55 +++-- .../query/ConnectionlessQueryServicesImpl.java | 28 +-- .../query/DelegateConnectionQueryServices.java | 12 +- .../apache/phoenix/query/GuidePostsCache.java | 231 +++++++++++++++++++ .../apache/phoenix/query/QueryConstants.java | 2 +- .../apache/phoenix/query/TableStatsCache.java | 192 --------------- .../apache/phoenix/schema/MetaDataClient.java | 49 ++-- .../stats/DefaultStatisticsCollector.java | 44 ++-- .../phoenix/schema/stats/GuidePostsInfo.java | 77 ++++--- .../schema/stats/GuidePostsInfoBuilder.java | 4 + .../phoenix/schema/stats/GuidePostsKey.java | 84 +++++++ .../schema/stats/NoOpStatisticsCollector.java | 3 +- .../phoenix/schema/stats/PTableStats.java | 57 ----- .../phoenix/schema/stats/PTableStatsImpl.java | 115 --------- .../schema/stats/StatisticsCollector.java | 5 +- .../phoenix/schema/stats/StatisticsUtil.java | 96 ++++---- .../phoenix/schema/stats/StatisticsWriter.java | 6 +- .../org/apache/phoenix/util/MetaDataUtil.java | 4 + .../phoenix/filter/SkipScanBigFilterTest.java | 25 +- .../PhoenixStatsCacheRemovalListenerTest.java | 2 +- .../java/org/apache/phoenix/util/TestUtil.java | 28 ++- 32 files changed, 697 insertions(+), 696 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java index 56f4503..1b32cfc 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java @@ -33,6 +33,7 @@ import java.sql.Statement; import java.util.List; import java.util.Properties; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.AmbiguousColumnException; import org.apache.phoenix.schema.types.PChar; @@ -409,11 +410,11 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); String tableName = generateUniqueName(); - initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 "); + initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 "); testAvgGroupByOrderPreserving(conn, tableName, 13); - conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100); + conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100"); testAvgGroupByOrderPreserving(conn, tableName, 6); - conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null"); + conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null"); testAvgGroupByOrderPreserving(conn, tableName, 4); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java index cfbc9eb..93d7cec 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java @@ -92,10 +92,11 @@ public class IndexExtendedIT extends BaseOwnClusterIT { @BeforeClass public static void doSetup() throws Exception { - Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1); + Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2); serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(1); + Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet() .iterator())); } @@ -442,7 +443,7 @@ public class IndexExtendedIT extends BaseOwnClusterIT { rs = conn1.createStatement().executeQuery(query); Thread.sleep(1000); for (int j = 0; j < 26; j++) { - assertTrue(rs.next()); + assertTrue("No row found at " + j, rs.next()); tIdColumnValues[j] = rs.getString("t_id"); k1ColumnValue[j] = rs.getInt("k1"); v1ColumnValues[j] = rs.getString("V1"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java index c247bc9..9961199 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java @@ -175,7 +175,6 @@ public class RoundFloorCeilFuncIT extends ParallelStatsDisabledIT { expectedDate = DateUtil.parseDate("2012-01-01 14:25:29"); assertEquals(expectedDate, rs.getDate(4)); expectedDate = DateUtil.parseDate("2012-01-02 00:00:00"); - System.out.println(String.format(" the expected time is [%s] and the actual time is [%s]",expectedDate.getTime(),rs.getDate(5).getTime())); assertEquals(expectedDate, rs.getDate(5)); expectedDate = DateUtil.parseDate("2012-02-01 00:00:00"); assertEquals(expectedDate, rs.getDate(6)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java index 2445948..6193cad 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java @@ -40,13 +40,15 @@ import java.util.Random; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; @@ -69,10 +71,14 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { } private static Connection getConnection() throws SQLException { + return getConnection(Integer.MAX_VALUE); + } + + private static Connection getConnection(Integer statsUpdateFreq) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString()); props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString()); - props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); + props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Integer.toString(statsUpdateFreq)); return DriverManager.getConnection(getUrl(), props); } @@ -334,7 +340,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { @Test public void testCompactUpdatesStats() throws Exception { - testCompactUpdatesStats(null, fullTableName); + testCompactUpdatesStats(0, fullTableName); } @Test @@ -342,9 +348,17 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, fullTableName); } - private void testCompactUpdatesStats(Integer minStatsUpdateFreq, String tableName) throws Exception { + private static void invalidateStats(Connection conn, String tableName) throws SQLException { + PTable ptable = conn.unwrap(PhoenixConnection.class) + .getMetaDataCache().getTableRef(new PTableKey(null, tableName)) + .getTable(); + byte[] name = ptable.getPhysicalName().getBytes(); + conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(new GuidePostsKey(name, SchemaUtil.getEmptyColumnFamily(ptable))); + } + + private void testCompactUpdatesStats(Integer statsUpdateFreq, String tableName) throws Exception { int nRows = 10; - Connection conn = getConnection(); + Connection conn = getConnection(statsUpdateFreq); PreparedStatement stmt; conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER, w INTEGER) " + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE); @@ -358,9 +372,8 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { conn.commit(); compactTable(conn, tableName); - if (minStatsUpdateFreq == null) { - ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName)); - conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr); + if (statsUpdateFreq == null) { + invalidateStats(conn, tableName); } else { // Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run // UPDATATE STATISTICS, the new statistics are faulted in as expected. @@ -379,13 +392,12 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { assertEquals(5, nDeletedRows); compactTable(conn, tableName); - if (minStatsUpdateFreq == null) { - ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName)); - conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr); + if (statsUpdateFreq == null) { + invalidateStats(conn, tableName); } keyRanges = getAllSplits(conn, tableName); - if (minStatsUpdateFreq != null) { + if (statsUpdateFreq != null) { assertEquals(nRows+1, keyRanges.size()); // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache // and force us to pull over the new stats @@ -403,7 +415,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { @Test public void testWithMultiCF() throws Exception { int nRows = 20; - Connection conn = getConnection(); + Connection conn = getConnection(0); PreparedStatement stmt; conn.createStatement().execute( "CREATE TABLE " + fullTableName @@ -478,6 +490,20 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { assertEquals(6, rs.getInt(4)); assertFalse(rs.next()); + + // Disable stats + conn.createStatement().execute("ALTER TABLE " + fullTableName + + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=0"); + TestUtil.analyzeTable(conn, fullTableName); + // Assert that there are no more guideposts + rs = conn.createStatement().executeQuery("SELECT count(1) FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + "='" + fullTableName + "' AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NOT NULL"); + assertTrue(rs.next()); + assertEquals(0, rs.getLong(1)); + assertFalse(rs.next()); + rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName); + assertEquals("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER " + fullTableName, + QueryUtil.getExplainPlan(rs)); } @Test @@ -506,7 +532,6 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT { int endIndex = r.nextInt(strings.length - startIndex) + startIndex; long rows = endIndex - startIndex; long c2Bytes = rows * 35; - System.out.println(rows + ":" + startIndex + ":" + endIndex); rs = conn.createStatement().executeQuery( "SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH) from SYSTEM.STATS where PHYSICAL_NAME = '" + fullTableName + "' AND GUIDE_POST_KEY>= cast('" + strings[startIndex] http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index d740013..d6c1e9c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@ -377,9 +377,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterIT { conn.commit(); fail(); } catch (SQLException e) { - System.out.println(); } catch (Exception e) { - System.out.println(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index 1bc01d2..38c9f43 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@ -706,7 +706,6 @@ public class MutableIndexIT extends ParallelStatsDisabledIT { while (true) { rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); - System.out.println("Number of rows returned:" + rs.getInt(1)); assertEquals(4, rs.getInt(1)); //TODO this returns 5 sometimes instead of 4, duplicate results? try { List<HRegionInfo> indexRegions = admin.getTableRegions(indexTable); http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java index b714a11..9e63093 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java @@ -31,11 +31,14 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Properties; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTableType; @@ -145,19 +148,19 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { stmt.setString(4, "x1"); stmt.setInt(5, 100); stmt.execute(); - stmt.setString(1, "20"); + stmt.setString(1, "10"); stmt.setString(2, "b"); stmt.setInt(3, 2); stmt.setString(4, "x2"); stmt.setInt(5, 200); stmt.execute(); - stmt.setString(1, "30"); + stmt.setString(1, "10"); stmt.setString(2, "c"); stmt.setInt(3, 3); stmt.setString(4, "x3"); stmt.setInt(5, 300); stmt.execute(); - stmt.setString(1, "40"); + stmt.setString(1, "20"); stmt.setString(2, "d"); stmt.setInt(3, 4); stmt.setString(4, "x4"); @@ -185,24 +188,24 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); assertFalse(rs.next()); -// TestUtil.analyzeTable(conn, fullTableName); -// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName); -// assertEquals(1, guideposts.size()); -// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); -// -// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20"); -// -// TestUtil.analyzeTable(conn, fullTableName); -// guideposts = TestUtil.getAllSplits(conn, fullTableName); -// assertEquals(5, guideposts.size()); -// -// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table -// // was used -// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100"; -// stmt = conn1.prepareStatement(sql); -// stmt.executeQuery(); -// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); -// assertEquals(5, plan.getSplits().size()); + TestUtil.analyzeTable(conn, fullTableName); + List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName); + assertEquals(1, guideposts.size()); + assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); + + conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20"); + + TestUtil.analyzeTable(conn, fullTableName); + guideposts = TestUtil.getAllSplits(conn, fullTableName); + assertEquals(5, guideposts.size()); + + // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table + // was used + sql = "SELECT * FROM " + viewName + " WHERE v2 >= 100"; + stmt = conn1.prepareStatement(sql); + stmt.executeQuery(); + QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); + assertEquals(4, plan.getSplits().size()); } @@ -259,26 +262,26 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { assertEquals("KV1", rs.getString(1)); assertFalse(rs.next()); -// TestUtil.analyzeTable(conn, baseTable); -// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable); -// assertEquals(1, guideposts.size()); -// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); -// -// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20"); -// -// TestUtil.analyzeTable(conn, baseTable); -// guideposts = TestUtil.getAllSplits(conn, baseTable); -// assertEquals(6, guideposts.size()); -// -// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table -// // was used -// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?"); -// stmt.setInt(1, 1); -// stmt.setString(2, "KV3"); -// rs = stmt.executeQuery(); -// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); -// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx)); -// assertEquals(6, plan.getSplits().size()); + TestUtil.analyzeTable(conn, baseTable); + List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable); + assertEquals(1, guideposts.size()); + assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); + + conn.createStatement().execute("ALTER TABLE " + baseTable + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20"); + + TestUtil.analyzeTable(conn, baseTable); + guideposts = TestUtil.getAllSplits(conn, baseTable); + assertEquals(6, guideposts.size()); + + // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table + // was used + stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?"); + stmt.setInt(1, 1); + stmt.setString(2, "KV3"); + rs = stmt.executeQuery(); + plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); + assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx)); + assertEquals(6, plan.getSplits().size()); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 2a4bfca..d8d313d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -718,7 +718,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsCollector stats, final Region region, final Scan scan, Configuration config) throws IOException { StatsCollectionCallable callable = - new StatsCollectionCallable(stats, region, innerScanner, config); + new StatsCollectionCallable(stats, region, innerScanner, config, scan); byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB); boolean async = false; if (asyncBytes != null) { @@ -785,13 +785,15 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver private final Region region; private final RegionScanner innerScanner; private final Configuration config; + private final Scan scan; StatsCollectionCallable(StatisticsCollector s, Region r, RegionScanner rs, - Configuration config) { + Configuration config, Scan scan) { this.stats = s; this.region = r; this.innerScanner = rs; this.config = config; + this.scan = scan; } @Override @@ -832,7 +834,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver } finally { try { if (noErrors && !compactionRunning) { - stats.updateStatistic(region); + stats.updateStatistic(region, scan); logger.info("UPDATE STATISTICS finished successfully for scanner: " + innerScanner + ". Number of rows scanned: " + rowCount + ". Time: " + (System.currentTimeMillis() - startTime)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 2685b93..d4c8bef 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -86,7 +86,7 @@ import org.apache.phoenix.schema.PTable.ViewType; import org.apache.phoenix.schema.StaleRegionBoundaryCacheException; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.GuidePostsInfo; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.schema.stats.StatisticsUtil; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.Closeables; @@ -120,7 +120,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result private final List<List<Scan>> scans; private final List<KeyRange> splits; - private final PTableStats tableStats; private final byte[] physicalTableName; protected final QueryPlan plan; protected final String scanId; @@ -363,9 +362,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result if (null == currentSCN) { currentSCN = HConstants.LATEST_TIMESTAMP; } - tableStats = useStats() && StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName)) - ? context.getConnection().getQueryServices().getTableStats(physicalTableName, currentSCN) - : PTableStats.EMPTY_STATS; // Used to tie all the scans together during logging scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString(); @@ -425,55 +421,40 @@ public abstract class BaseResultIterators extends ExplainTable implements Result return guideIndex; } - private GuidePostsInfo getGuidePosts(Set<byte[]> whereConditions) { - if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; } + private GuidePostsInfo getGuidePosts() throws SQLException { + if (!useStats() || !StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName))) { + return GuidePostsInfo.NO_GUIDEPOST; + } - GuidePostsInfo gps = null; + TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); + for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) { + byte[] cf = where.getFirst(); + if (cf != null) { + whereConditions.add(cf); + } + } PTable table = getTable(); - Map<byte[], GuidePostsInfo> guidePostMap = tableStats.getGuidePosts(); byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); - if (table.getColumnFamilies().isEmpty()) { - // For sure we can get the defaultCF from the table - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } else { - if (whereConditions.isEmpty() || whereConditions.contains(defaultCF)) { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } else { - byte[] familyInWhere = whereConditions.iterator().next(); - GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); - if (guidePostsInfo != null) { - gps = guidePostsInfo; - } else { - // As there are no guideposts collected for the where family we go with the default CF - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + byte[] cf = null; + if ( !table.getColumnFamilies().isEmpty() && !whereConditions.isEmpty() ) { + for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) { + byte[] whereCF = where.getFirst(); + if (Bytes.compareTo(defaultCF, whereCF) == 0) { + cf = defaultCF; + break; } } + if (cf == null) { + cf = context.getWhereConditionColumns().get(0).getFirst(); + } } - if (gps == null) { return GuidePostsInfo.NO_GUIDEPOST; } - return gps; - } - - private GuidePostsInfo getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) { - if (guidePostMap.get(defaultCF) != null) { - return guidePostMap.get(defaultCF); + if (cf == null) { + cf = defaultCF; } - return null; + GuidePostsKey key = new GuidePostsKey(physicalTableName, cf); + return context.getConnection().getQueryServices().getTableStats(key); } - private static String toString(List<byte[]> gps) { - StringBuilder buf = new StringBuilder(gps.size() * 100); - buf.append("["); - for (int i = 0; i < gps.size(); i++) { - buf.append(Bytes.toStringBinary(gps.get(i))); - buf.append(","); - if (i > 0 && i < gps.size()-1 && (i % 10) == 0) { - buf.append("\n"); - } - } - buf.setCharAt(buf.length()-1, ']'); - return buf.toString(); - } - private List<Scan> addNewScan(List<List<Scan>> parallelScans, List<Scan> scans, Scan scan, byte[] startKey, boolean crossedRegionBoundary, HRegionLocation regionLocation) { boolean startNewScan = scanGrouper.shouldStartNewScan(plan, scans, startKey, crossedRegionBoundary); if (scan != null) { @@ -565,14 +546,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result PTable table = getTable(); boolean isSalted = table.getBucketNum() != null; boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL; - TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); - for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) { - byte[] cf = where.getFirst(); - if (cf != null) { - whereConditions.add(cf); - } - } - GuidePostsInfo gps = getGuidePosts(whereConditions); + GuidePostsInfo gps = getGuidePosts(); hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST; boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { @@ -649,8 +623,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result if (newScan != null) { ScanUtil.setLocalIndexAttributes(newScan, keyOffset, regionInfo.getStartKey(), regionInfo.getEndKey(), newScan.getStartRow(), newScan.getStopRow()); - estimatedRows += gps.getRowCounts().get(guideIndex); - estimatedSize += gps.getByteCounts().get(guideIndex); + estimatedRows += gps.getRowCounts()[guideIndex]; + estimatedSize += gps.getByteCounts()[guideIndex]; } scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); currentKeyBytes = currentGuidePostBytes; http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 47dfd4e..35b754f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -1087,7 +1087,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { SQLViewTypeFunction.NAME + "(" + VIEW_TYPE + ") AS " + VIEW_TYPE + "," + SQLIndexTypeFunction.NAME + "(" + INDEX_TYPE + ") AS " + INDEX_TYPE + "," + TRANSACTIONAL + "," + - IS_NAMESPACE_MAPPED + + IS_NAMESPACE_MAPPED + "," + + GUIDE_POSTS_WIDTH + " from " + SYSTEM_CATALOG + " " + SYSTEM_CATALOG_ALIAS + " where " + COLUMN_NAME + " is null" + " and " + COLUMN_FAMILY + " is null" + @@ -1126,7 +1127,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { "'' " + VIEW_TYPE + "," + "'' " + INDEX_TYPE + "," + "CAST(null AS BOOLEAN) " + TRANSACTIONAL + "," + - "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "\n"); + "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "," + + "CAST(null AS BIGINT) " + GUIDE_POSTS_WIDTH + "\n"); buf.append( " from " + SYSTEM_SEQUENCE + "\n"); StringBuilder whereClause = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java index bc2c93b..0478e07 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.MutationPlan; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.execute.MutationState; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.parse.PFunction; @@ -43,7 +42,8 @@ import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceAllocation; import org.apache.phoenix.schema.SequenceKey; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.tephra.TransactionSystemClient; @@ -118,7 +118,14 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated public String getUserName(); public void clearTableFromCache(final byte[] tenantId, final byte[] schemaName, final byte[] tableName, long clientTS) throws SQLException; - public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException; + public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException; + /** + * Removes cache {@link GuidePostsInfo} for the table with the given name. If no cached guideposts are present, this does nothing. + * + * @param tableName The table to remove stats for + */ + void invalidateStats(GuidePostsKey key); + public long clearCache() throws SQLException; public int getSequenceSaltBuckets(); @@ -133,13 +140,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated public MetaDataMutationResult dropSchema(List<Mutation> schemaMetaData, String schemaName) throws SQLException; - /** - * Removes cache {@link PTableStats} for the table with the given name. If no cached stats are present, this does nothing. - * - * @param tableName The table to remove stats for - */ - void invalidateStats(ImmutableBytesPtr tableName); - boolean isUpgradeRequired(); void upgradeSystemTables(String url, Properties props) throws SQLException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 4982578..4770d22 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -140,7 +140,6 @@ import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy; import org.apache.phoenix.hbase.index.Indexer; import org.apache.phoenix.hbase.index.covered.NonTxIndexBuilder; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.hbase.index.util.VersionUtil; import org.apache.phoenix.index.PhoenixIndexBuilder; @@ -179,7 +178,8 @@ import org.apache.phoenix.schema.SequenceKey; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableProperty; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; @@ -235,7 +235,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement private final ReadOnlyProps props; private final String userName; private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices; - private final TableStatsCache tableStatsCache; + private final GuidePostsCache tableStatsCache; // Cache the latest meta data here for future connections // writes guarded by "latestMetaDataLock" @@ -348,7 +348,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } connectionQueues = ImmutableList.copyOf(list); // A little bit of a smell to leak `this` here, but should not be a problem - this.tableStatsCache = new TableStatsCache(this, config); + this.tableStatsCache = new GuidePostsCache(this, config); this.isAutoUpgradeEnabled = config.getBoolean(AUTO_UPGRADE_ENABLED, QueryServicesOptions.DEFAULT_AUTO_UPGRADE_ENABLED); } @@ -1317,13 +1317,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement private boolean ensureViewIndexTableDropped(byte[] physicalTableName, long timestamp) throws SQLException { byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName); - HTableDescriptor desc = null; boolean wasDeleted = false; try (HBaseAdmin admin = getAdmin()) { try { - desc = admin.getTableDescriptor(physicalIndexName); + HTableDescriptor desc = admin.getTableDescriptor(physicalIndexName); if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) { - this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalIndexName)); final ReadOnlyProps props = this.getProps(); final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); if (dropMetadata) { @@ -1331,6 +1329,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement admin.deleteTable(physicalIndexName); clearTableRegionCache(physicalIndexName); wasDeleted = true; + } else { + this.tableStatsCache.invalidateAll(desc); } } } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) { @@ -1348,7 +1348,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement try (HBaseAdmin admin = getAdmin()) { try { desc = admin.getTableDescriptor(physicalTableName); - this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalTableName)); + for (byte[] fam : desc.getFamiliesKeys()) { + this.tableStatsCache.invalidate(new GuidePostsKey(physicalTableName, fam)); + } final ReadOnlyProps props = this.getProps(); final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); if (dropMetadata) { @@ -1530,14 +1532,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (dropMetadata) { flushParentPhysicalTable(table); dropTables(result.getTableNamesToDelete()); + } else { + invalidateTableStats(result.getTableNamesToDelete()); } - invalidateTables(result.getTableNamesToDelete()); long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData); if (tableType == PTableType.TABLE) { byte[] physicalName = table.getPhysicalName().getBytes(); ensureViewIndexTableDropped(physicalName, timestamp); ensureLocalIndexTableDropped(physicalName, timestamp); - tableStatsCache.invalidate(new ImmutableBytesPtr(physicalName)); + tableStatsCache.invalidateAll(table); } break; default: @@ -1598,24 +1601,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement return result; } - private void invalidateTables(final List<byte[]> tableNamesToDelete) { + private void invalidateTableStats(final List<byte[]> tableNamesToDelete) { if (tableNamesToDelete != null) { for (byte[] tableName : tableNamesToDelete) { - tableStatsCache.invalidate(new ImmutableBytesPtr(Bytes.toString(tableName) - .replace(QueryConstants.NAMESPACE_SEPARATOR, QueryConstants.NAME_SEPARATOR).getBytes())); + tableStatsCache.invalidateAll(tableName); } } } + private void dropTable(byte[] tableNameToDelete) throws SQLException { + dropTables(Collections.<byte[]>singletonList(tableNameToDelete)); + } + private void dropTables(final List<byte[]> tableNamesToDelete) throws SQLException { SQLException sqlE = null; try (HBaseAdmin admin = getAdmin()) { if (tableNamesToDelete != null){ for ( byte[] tableName : tableNamesToDelete ) { - if ( admin.tableExists(tableName) ) { + try { + HTableDescriptor htableDesc = this.getTableDescriptor(tableName); admin.disableTable(tableName); admin.deleteTable(tableName); + tableStatsCache.invalidateAll(htableDesc); clearTableRegionCache(tableName); + } catch (TableNotFoundException ignore) { } } } @@ -2248,8 +2257,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); if (dropMetadata) { dropTables(result.getTableNamesToDelete()); + } else { + invalidateTableStats(result.getTableNamesToDelete()); } - invalidateTables(result.getTableNamesToDelete()); break; default: break; @@ -3478,7 +3488,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement sqlE = new SQLException(e); } finally { try { - if (tenantId.length == 0) tableStatsCache.invalidate(new ImmutableBytesPtr(SchemaUtil.getTableNameAsBytes(schemaName, tableName))); htable.close(); } catch (IOException e) { if (sqlE == null) { @@ -3680,9 +3689,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } @Override - public PTableStats getTableStats(final byte[] physicalName, final long clientTimeStamp) throws SQLException { + public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException { try { - return tableStatsCache.get(new ImmutableBytesPtr(physicalName)); + return tableStatsCache.get(key); } catch (ExecutionException e) { throw ServerUtil.parseServerException(e); } @@ -4042,19 +4051,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } /** - * Manually adds {@link PTableStats} for a table to the client-side cache. Not a + * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a * {@link ConnectionQueryServices} method. Exposed for testing purposes. * * @param tableName Table name * @param stats Stats instance */ - public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) { - this.tableStatsCache.put(Objects.requireNonNull(tableName), stats); + public void addTableStats(GuidePostsKey key, GuidePostsInfo info) { + this.tableStatsCache.put(Objects.requireNonNull(key), Objects.requireNonNull(info)); } @Override - public void invalidateStats(ImmutableBytesPtr tableName) { - this.tableStatsCache.invalidate(Objects.requireNonNull(tableName)); + public void invalidateStats(GuidePostsKey key) { + this.tableStatsCache.invalidate(Objects.requireNonNull(key)); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 8c9f3ff..1b1e429 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -51,7 +51,6 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; @@ -79,7 +78,8 @@ import org.apache.phoenix.schema.SequenceKey; import org.apache.phoenix.schema.SequenceNotFoundException; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.util.JDBCUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; @@ -113,7 +113,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple private volatile boolean initialized; private volatile SQLException initializationException; private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap(); - private final TableStatsCache tableStatsCache; + private final GuidePostsCache guidePostsCache; public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) { super(services); @@ -140,7 +140,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config); TransactionManager txnManager = new TransactionManager(config); this.txSystemClient = new InMemoryTxSystemClient(txnManager); - this.tableStatsCache = new TableStatsCache(this, config); + this.guidePostsCache = new GuidePostsCache(this, config); } private PMetaData newEmptyMetaData() { @@ -520,12 +520,12 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple } @Override - public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) { - PTableStats stats = tableStatsCache.getCache().getIfPresent(physicalName); - if (null == stats) { - return PTableStats.EMPTY_STATS; + public GuidePostsInfo getTableStats(GuidePostsKey key) { + GuidePostsInfo info = guidePostsCache.getCache().getIfPresent(key); + if (null == info) { + return GuidePostsInfo.NO_GUIDEPOST; } - return stats; + return info; } @Override @@ -640,19 +640,19 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple } /** - * Manually adds {@link PTableStats} for a table to the client-side cache. Not a + * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a * {@link ConnectionQueryServices} method. Exposed for testing purposes. * * @param tableName Table name * @param stats Stats instance */ - public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) { - this.tableStatsCache.put(Objects.requireNonNull(tableName), stats); + public void addTableStats(GuidePostsKey key, GuidePostsInfo info) { + this.guidePostsCache.put(Objects.requireNonNull(key), info); } @Override - public void invalidateStats(ImmutableBytesPtr tableName) { - this.tableStatsCache.invalidate(Objects.requireNonNull(tableName)); + public void invalidateStats(GuidePostsKey key) { + this.guidePostsCache.invalidate(Objects.requireNonNull(key)); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java index b00e92b..7466e9c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.MutationPlan; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.execute.MutationState; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.parse.PFunction; @@ -45,7 +44,8 @@ import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceAllocation; import org.apache.phoenix.schema.SequenceKey; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.tephra.TransactionSystemClient; @@ -250,8 +250,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple } @Override - public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException { - return getDelegate().getTableStats(physicalName, clientTimeStamp); + public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException { + return getDelegate().getTableStats(key); } @@ -342,8 +342,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple } @Override - public void invalidateStats(ImmutableBytesPtr tableName) { - getDelegate().invalidateStats(tableName); + public void invalidateStats(GuidePostsKey key) { + getDelegate().invalidateStats(key); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java new file mode 100644 index 0000000..d27be1b --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.query; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.schema.PColumnFamily; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsKey; +import org.apache.phoenix.schema.stats.StatisticsUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalCause; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; +import com.google.common.cache.Weigher; + +/** + * "Client-side" cache for storing {@link GuidePostsInfo} for a column family. Intended to decouple + * Phoenix from a specific version of Guava's cache. + */ +public class GuidePostsCache { + private static final Logger logger = LoggerFactory.getLogger(GuidePostsCache.class); + + private final ConnectionQueryServices queryServices; + private final LoadingCache<GuidePostsKey, GuidePostsInfo> cache; + + public GuidePostsCache(ConnectionQueryServices queryServices, Configuration config) { + this.queryServices = Objects.requireNonNull(queryServices); + // Number of millis to expire cache values after write + final long statsUpdateFrequency = config.getLong( + QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS); + // Maximum number of entries (tables) to store in the cache at one time + final long maxTableStatsCacheSize = config.getLong( + QueryServices.STATS_MAX_CACHE_SIZE, + QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE); + cache = CacheBuilder.newBuilder() + // Expire entries a given amount of time after they were written + .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS) + // Maximum total weight (size in bytes) of stats entries + .maximumWeight(maxTableStatsCacheSize) + // Defer actual size to the PTableStats.getEstimatedSize() + .weigher(new Weigher<GuidePostsKey, GuidePostsInfo>() { + @Override public int weigh(GuidePostsKey key, GuidePostsInfo info) { + return info.getEstimatedSize(); + } + }) + // Log removals at TRACE for debugging + .removalListener(new PhoenixStatsCacheRemovalListener()) + // Automatically load the cache when entries are missing + .build(new StatsLoader()); + } + + /** + * {@link CacheLoader} implementation for the Phoenix Table Stats cache. + */ + protected class StatsLoader extends CacheLoader<GuidePostsKey, GuidePostsInfo> { + @Override + public GuidePostsInfo load(GuidePostsKey statsKey) throws Exception { + @SuppressWarnings("deprecation") + HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName( + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, + queryServices.getProps()).getName()); + try { + GuidePostsInfo guidePostsInfo = StatisticsUtil.readStatistics(statsHTable, statsKey, + HConstants.LATEST_TIMESTAMP); + traceStatsUpdate(statsKey, guidePostsInfo); + return guidePostsInfo; + } catch (TableNotFoundException e) { + // On a fresh install, stats might not yet be created, don't warn about this. + logger.debug("Unable to locate Phoenix stats table", e); + return GuidePostsInfo.NO_GUIDEPOST; + } catch (IOException e) { + logger.warn("Unable to read from stats table", e); + // Just cache empty stats. We'll try again after some time anyway. + return GuidePostsInfo.NO_GUIDEPOST; + } finally { + try { + statsHTable.close(); + } catch (IOException e) { + // Log, but continue. We have our stats anyway now. + logger.warn("Unable to close stats table", e); + } + } + } + + /** + * Logs a trace message for newly inserted entries to the stats cache. + */ + void traceStatsUpdate(GuidePostsKey key, GuidePostsInfo info) { + if (logger.isTraceEnabled()) { + logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes", + new Object[] {Objects.hashCode(GuidePostsCache.this), key, + info.getEstimatedSize()}); + } + } + } + + /** + * Returns the underlying cache. Try to use the provided methods instead of accessing the cache + * directly. + */ + LoadingCache<GuidePostsKey, GuidePostsInfo> getCache() { + return cache; + } + + /** + * Returns the PTableStats for the given <code>tableName</code, using the provided + * <code>valueLoader</code> if no such mapping exists. + * + * @see com.google.common.cache.LoadingCache#get(Object) + */ + public GuidePostsInfo get(GuidePostsKey key) throws ExecutionException { + return getCache().get(key); + } + + /** + * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>. + * + * @see com.google.common.cache.Cache#put(Object, Object) + */ + public void put(GuidePostsKey key, GuidePostsInfo info) { + getCache().put(Objects.requireNonNull(key), Objects.requireNonNull(info)); + } + + /** + * Removes the mapping for <code>tableName</code> if it exists. + * + * @see com.google.common.cache.Cache#invalidate(Object) + */ + public void invalidate(GuidePostsKey key) { + getCache().invalidate(Objects.requireNonNull(key)); + } + + /** + * Removes all mappings from the cache. + * + * @see com.google.common.cache.Cache#invalidateAll() + */ + public void invalidateAll() { + getCache().invalidateAll(); + } + + /** + * Removes all mappings where the {@link org.apache.phoenix.schema.stats.GuidePostsKey#getPhysicalName()} + * equals physicalName. Because all keys in the map must be iterated, this method should be avoided. + * @param physicalName + */ + public void invalidateAll(byte[] physicalName) { + for (GuidePostsKey key : getCache().asMap().keySet()) { + if (Bytes.compareTo(key.getPhysicalName(), physicalName) == 0) { + invalidate(key); + } + } + } + + public void invalidateAll(HTableDescriptor htableDesc) { + byte[] tableName = htableDesc.getTableName().getName(); + for (byte[] fam : htableDesc.getFamiliesKeys()) { + invalidate(new GuidePostsKey(tableName, fam)); + } + } + + public void invalidateAll(PTable table) { + byte[] physicalName = table.getPhysicalName().getBytes(); + List<PColumnFamily> families = table.getColumnFamilies(); + if (families.isEmpty()) { + invalidate(new GuidePostsKey(physicalName, SchemaUtil.getEmptyColumnFamily(table))); + } else { + for (PColumnFamily family : families) { + invalidate(new GuidePostsKey(physicalName, family.getName().getBytes())); + } + } + } + + /** + * A {@link RemovalListener} implementation to track evictions from the table stats cache. + */ + static class PhoenixStatsCacheRemovalListener implements + RemovalListener<GuidePostsKey, GuidePostsInfo> { + @Override + public void onRemoval(RemovalNotification<GuidePostsKey, GuidePostsInfo> notification) { + if (logger.isTraceEnabled()) { + final RemovalCause cause = notification.getCause(); + if (wasEvicted(cause)) { + GuidePostsKey key = notification.getKey(); + logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}", + new Object[] {key, notification.getValue().getEstimatedSize(), + cause}); + } + } + } + + boolean wasEvicted(RemovalCause cause) { + // This is actually a method on RemovalCause but isn't exposed + return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause; + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index fd37328..89f7aba 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -301,7 +301,7 @@ public interface QueryConstants { "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" + // PK columns PHYSICAL_NAME + " VARCHAR NOT NULL," + - COLUMN_FAMILY + " VARCHAR NOT NULL," + + COLUMN_FAMILY + " VARCHAR," + GUIDE_POST_KEY + " VARBINARY," + GUIDE_POSTS_WIDTH + " BIGINT," + LAST_STATS_UPDATE_TIME+ " DATE, "+ http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java deleted file mode 100644 index 2c7b2db..0000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.query; - -import java.io.IOException; -import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; -import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.schema.stats.PTableStats; -import org.apache.phoenix.schema.stats.StatisticsUtil; -import org.apache.phoenix.util.SchemaUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalCause; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.cache.Weigher; - -/** - * "Client-side" cache for storing {@link PTableStats} for Phoenix tables. Intended to decouple - * Phoenix from a specific version of Guava's cache. - */ -public class TableStatsCache { - private static final Logger logger = LoggerFactory.getLogger(TableStatsCache.class); - - private final ConnectionQueryServices queryServices; - private final LoadingCache<ImmutableBytesPtr, PTableStats> cache; - - public TableStatsCache(ConnectionQueryServices queryServices, Configuration config) { - this.queryServices = Objects.requireNonNull(queryServices); - // Number of millis to expire cache values after write - final long statsUpdateFrequency = config.getLong( - QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS); - // Maximum number of entries (tables) to store in the cache at one time - final long maxTableStatsCacheSize = config.getLong( - QueryServices.STATS_MAX_CACHE_SIZE, - QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE); - cache = CacheBuilder.newBuilder() - // Expire entries a given amount of time after they were written - .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS) - // Maximum total weight (size in bytes) of stats entries - .maximumWeight(maxTableStatsCacheSize) - // Defer actual size to the PTableStats.getEstimatedSize() - .weigher(new Weigher<ImmutableBytesPtr, PTableStats>() { - @Override public int weigh(ImmutableBytesPtr key, PTableStats stats) { - return stats.getEstimatedSize(); - } - }) - // Log removals at TRACE for debugging - .removalListener(new PhoenixStatsCacheRemovalListener()) - // Automatically load the cache when entries are missing - .build(new StatsLoader()); - } - - /** - * {@link CacheLoader} implementation for the Phoenix Table Stats cache. - */ - protected class StatsLoader extends CacheLoader<ImmutableBytesPtr, PTableStats> { - @Override - public PTableStats load(ImmutableBytesPtr tableName) throws Exception { - @SuppressWarnings("deprecation") - HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName( - PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES, - queryServices.getProps()).getName()); - final byte[] tableNameBytes = tableName.copyBytesIfNecessary(); - try { - PTableStats stats = StatisticsUtil.readStatistics(statsHTable, tableNameBytes, - Long.MAX_VALUE); - traceStatsUpdate(tableNameBytes, stats); - return stats; - } catch (TableNotFoundException e) { - // On a fresh install, stats might not yet be created, don't warn about this. - logger.debug("Unable to locate Phoenix stats table", e); - return PTableStats.EMPTY_STATS; - } catch (IOException e) { - logger.warn("Unable to read from stats table", e); - // Just cache empty stats. We'll try again after some time anyway. - return PTableStats.EMPTY_STATS; - } finally { - try { - statsHTable.close(); - } catch (IOException e) { - // Log, but continue. We have our stats anyway now. - logger.warn("Unable to close stats table", e); - } - } - } - - /** - * Logs a trace message for newly inserted entries to the stats cache. - */ - void traceStatsUpdate(byte[] tableName, PTableStats stats) { - logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes", - new Object[] {Objects.hashCode(TableStatsCache.this), Bytes.toString(tableName), - stats.getEstimatedSize()}); - } - } - - /** - * Returns the underlying cache. Try to use the provided methods instead of accessing the cache - * directly. - */ - LoadingCache<ImmutableBytesPtr, PTableStats> getCache() { - return cache; - } - - /** - * Returns the PTableStats for the given <code>tableName</code, using the provided - * <code>valueLoader</code> if no such mapping exists. - * - * @see com.google.common.cache.LoadingCache#get(Object) - */ - public PTableStats get(ImmutableBytesPtr tableName) throws ExecutionException { - return getCache().get(tableName); - } - - /** - * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>. - * - * @see com.google.common.cache.Cache#put(Object, Object) - */ - public void put(ImmutableBytesPtr tableName, PTableStats stats) { - getCache().put(Objects.requireNonNull(tableName), Objects.requireNonNull(stats)); - } - - /** - * Removes the mapping for <code>tableName</code> if it exists. - * - * @see com.google.common.cache.Cache#invalidate(Object) - */ - public void invalidate(ImmutableBytesPtr tableName) { - getCache().invalidate(Objects.requireNonNull(tableName)); - } - - /** - * Removes all mappings from the cache. - * - * @see com.google.common.cache.Cache#invalidateAll() - */ - public void invalidateAll() { - getCache().invalidateAll(); - } - - /** - * A {@link RemovalListener} implementation to track evictions from the table stats cache. - */ - static class PhoenixStatsCacheRemovalListener implements - RemovalListener<ImmutableBytesPtr, PTableStats> { - @Override - public void onRemoval(RemovalNotification<ImmutableBytesPtr, PTableStats> notification) { - final RemovalCause cause = notification.getCause(); - if (wasEvicted(cause)) { - ImmutableBytesPtr ptr = notification.getKey(); - String tableName = new String(ptr.get(), ptr.getOffset(), ptr.getLength()); - logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}", - new Object[] {tableName, notification.getValue().getEstimatedSize(), - cause}); - } - } - - boolean wasEvicted(RemovalCause cause) { - // This is actually a method on RemovalCause but isn't exposed - return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause; - } - } -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 0c53d3e..54c6361 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -189,7 +189,7 @@ import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTable.ViewType; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PInteger; @@ -1085,9 +1085,20 @@ public class MetaDataClient { * This supports scenarios in which a major compaction was manually initiated and the * client wants the modified stats to be reflected immediately. */ - connection.getQueryServices().clearTableFromCache(tenantIdBytes, - Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())), - Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp); + if (cfs == null) { + List<PColumnFamily> families = logicalTable.getColumnFamilies(); + if (families.isEmpty()) { + connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), SchemaUtil.getEmptyColumnFamily(logicalTable))); + } else { + for (PColumnFamily family : families) { + connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), family.getName().getBytes())); + } + } + } else { + for (byte[] cf : cfs) { + connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), cf)); + } + } return rowCount; } @@ -1760,7 +1771,7 @@ public class MetaDataClient { updateCacheFrequency = updateCacheFrequencyProp; } String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps); - Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps); + Long guidePostsWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps); Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps); if (storeNullsProp == null) { @@ -2227,10 +2238,10 @@ public class MetaDataClient { tableUpsert.setString(23, autoPartitionSeq); } tableUpsert.setBoolean(24, isAppendOnlySchema); - if (guidePostWidth == null) { + if (guidePostsWidth == null) { tableUpsert.setNull(25, Types.BIGINT); } else { - tableUpsert.setLong(25, guidePostWidth); + tableUpsert.setLong(25, guidePostsWidth); } tableUpsert.execute(); @@ -3108,7 +3119,7 @@ public class MetaDataClient { connection.setAutoCommit(true); // Delete everything in the column. You'll still be able to do queries at earlier timestamps long ts = (scn == null ? result.getMutationTime() : scn); - MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts); + MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF == null ? null : Collections.singletonList(projectCF), null, ts); return connection.getQueryServices().updateData(plan); } return new MutationState(0,connection); @@ -3532,28 +3543,6 @@ public class MetaDataClient { connection.addSchema(result.getSchema()); } - public PTableStats getTableStats(PTable table) throws SQLException { - /* - * The shared view index case is tricky, because we don't have - * table meta data for it, only an HBase table. We do have stats, - * though, so we'll query them directly here and cache them so - * we don't keep querying for them. - */ - boolean isSharedIndex = table.getViewIndexId() != null; - if (isSharedIndex) { - // we are assuming the stats table is not transactional - return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), - getCurrentScn()); - } - boolean isView = table.getType() == PTableType.VIEW; - String physicalName = table.getPhysicalName().toString().replace(QueryConstants.NAMESPACE_SEPARATOR, - QueryConstants.NAME_SEPARATOR); - if (isView && table.getViewType() != ViewType.MAPPED) { - return connection.getQueryServices().getTableStats(Bytes.toBytes(physicalName), getCurrentScn()); - } - return connection.getQueryServices().getTableStats(table.getName().getBytes(), getCurrentScn()); - } - private void throwIfLastPKOfParentIsFixedLength(PTable parent, String viewSchemaName, String viewName, ColumnDef col) throws SQLException { if (isLastPKVariableLength(parent)) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MODIFY_VIEW_PK) http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java index 0cf5ed8..9cff48c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -64,7 +65,7 @@ class DefaultStatisticsCollector implements StatisticsCollector { private final Pair<Long, GuidePostsInfoBuilder> cachedGuidePosts; private final byte[] guidePostWidthBytes; private final byte[] guidePostPerRegionBytes; - // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG + // Where to look for GUIDE_POSTS_WIDTH in SYSTEM.CATALOG private final byte[] ptableKey; private final RegionCoprocessorEnvironment env; @@ -124,12 +125,12 @@ class DefaultStatisticsCollector implements StatisticsCollector { Get get = new Get(ptableKey); get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES); Result result = htable.get(get); - long guidepostWidth = 0; + long guidepostWidth = -1; if (!result.isEmpty()) { Cell cell = result.listCells().get(0); guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault()); } - if (guidepostWidth > 0) { + if (guidepostWidth >= 0) { this.guidePostDepth = guidepostWidth; } else { // Last use global config value @@ -157,10 +158,10 @@ class DefaultStatisticsCollector implements StatisticsCollector { } @Override - public void updateStatistic(Region region) { + public void updateStatistic(Region region, Scan scan) { try { ArrayList<Mutation> mutations = new ArrayList<Mutation>(); - writeStatsToStatsTable(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime()); + writeStatistics(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime(), scan); if (logger.isDebugEnabled()) { logger.debug("Committing new stats for the region " + region.getRegionInfo()); } @@ -170,16 +171,20 @@ class DefaultStatisticsCollector implements StatisticsCollector { } } - private void writeStatsToStatsTable(final Region region, boolean delete, List<Mutation> mutations, long currentTime) + private void writeStatistics(final Region region, boolean delete, List<Mutation> mutations, long currentTime, Scan scan) throws IOException { try { Set<ImmutableBytesPtr> fams = guidePostsInfoWriterMap.keySet(); - // update the statistics table - // Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS - // This will not impact a stats collection of single column family during compaction as - // guidePostsInfoWriterMap cannot be empty in this case. + // Update the statistics table. + // Delete statistics for a region if no guide posts are collected for that region during + // UPDATE STATISTICS. This will not impact a stats collection of single column family during + // compaction as guidePostsInfoWriterMap cannot be empty in this case. if (cachedGuidePosts == null) { - boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next()); + // We're either collecting stats for the data table or the local index table, but not both + // We can determine this based on the column families in the scan being prefixed with the + // local index column family prefix. We always explicitly specify the local index column + // families when we're collecting stats for a local index. + boolean collectingForLocalIndex = scan != null && !scan.getFamilyMap().isEmpty() && MetaDataUtil.isLocalIndexFamily(scan.getFamilyMap().keySet().iterator().next()); for (Store store : region.getStores()) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName()); boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey); @@ -202,7 +207,10 @@ class DefaultStatisticsCollector implements StatisticsCollector { if (logger.isDebugEnabled()) { logger.debug("Adding new stats for the region " + region.getRegionInfo()); } - statsWriter.addStats(this, fam, mutations); + // If we've disabled stats, don't write any, just delete them + if (this.guidePostDepth > 0) { + statsWriter.addStats(this, fam, mutations); + } } } catch (IOException e) { logger.error("Failed to update statistics table!", e); @@ -223,6 +231,10 @@ class DefaultStatisticsCollector implements StatisticsCollector { */ @Override public void collectStatistics(final List<Cell> results) { + // A guide posts depth of zero disables the collection of stats + if (guidePostDepth == 0) { + return; + } Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap(); boolean incrementRow = true; for (Cell cell : results) { @@ -274,8 +286,12 @@ class DefaultStatisticsCollector implements StatisticsCollector { } private InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, InternalScanner internalScan, - ImmutableBytesPtr family) { - return new StatisticsScanner(this, statsWriter, env, internalScan, family); + ImmutableBytesPtr family) throws IOException { + StatisticsScanner scanner = new StatisticsScanner(this, statsWriter, env, internalScan, family); + // We need to initialize the scanner synchronously and potentially perform a cross region Get + // in order to use the correct guide posts width for the table being compacted. + init(); + return scanner; } @Override
