Repository: phoenix Updated Branches: refs/heads/3.0 c89f2dd71 -> a5b797225
PHOENIX-1309 Ensure Phoenix table is created for Local index and view index tables to store guideposts against them (James Taylor via Ram) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a5b79722 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a5b79722 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a5b79722 Branch: refs/heads/3.0 Commit: a5b797225f4ab9df3a470b53ed0e73bd79822b00 Parents: c89f2dd Author: Ramkrishna <ramkrishna.s.vasude...@intel.com> Authored: Mon Oct 13 10:14:03 2014 +0530 Committer: Ramkrishna <ramkrishna.s.vasude...@intel.com> Committed: Mon Oct 13 10:14:03 2014 +0530 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/BaseViewIT.java | 2 + .../org/apache/phoenix/end2end/QueryIT.java | 20 +-- .../apache/phoenix/end2end/SaltedViewIT.java | 2 +- .../java/org/apache/phoenix/end2end/ViewIT.java | 11 +- .../UngroupedAggregateRegionObserver.java | 1 + .../phoenix/iterate/ParallelIterators.java | 86 +++++++---- .../phoenix/query/ConnectionQueryServices.java | 6 + .../query/ConnectionQueryServicesImpl.java | 41 ++++- .../query/ConnectionlessQueryServicesImpl.java | 14 ++ .../query/DelegateConnectionQueryServices.java | 16 ++ .../org/apache/phoenix/query/QueryServices.java | 1 + .../phoenix/query/QueryServicesOptions.java | 2 +- .../apache/phoenix/schema/MetaDataClient.java | 77 +++++++++- .../phoenix/schema/MetaDataClient.java.orig | 151 +++++++++++++++++-- .../apache/phoenix/schema/PColumnFamily.java | 2 - .../phoenix/schema/PColumnFamilyImpl.java | 19 --- .../java/org/apache/phoenix/schema/PTable.java | 7 - .../org/apache/phoenix/schema/PTableImpl.java | 21 +-- .../phoenix/schema/stats/PTableStats.java | 7 + .../phoenix/schema/stats/PTableStatsImpl.java | 41 +++++ .../schema/stats/StatisticsCollector.java | 14 +- .../phoenix/schema/stats/StatisticsWriter.java | 3 +- .../java/org/apache/phoenix/util/SizedUtil.java | 5 + .../phoenix/query/QueryServicesTestImpl.java | 2 - 24 files changed, 429 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java index e68c82e..b6a9994 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java @@ -27,6 +27,7 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.util.Map; +import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -43,6 +44,7 @@ public class BaseViewIT extends BaseHBaseManagedTimeIT { Map<String,String> props = Maps.newHashMapWithExpectedSize(1); // Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan // Must update config before starting server + props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Integer.toString(20)); setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java index bcc2973..ac1c61b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java @@ -204,7 +204,7 @@ public class QueryIT extends BaseQueryIT { } private void testNoStringValue(String value) throws Exception { - String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 1); + String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 10); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection upsertConn = DriverManager.getConnection(url, props); upsertConn.setAutoCommit(true); // Test auto commit @@ -215,13 +215,15 @@ public class QueryIT extends BaseQueryIT { stmt.setString(2, ROW5); stmt.setString(3, value); stmt.execute(); // should commit too + upsertConn.close(); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); Connection conn1 = DriverManager.getConnection(getUrl(), props); analyzeTable(conn1, "ATABLE"); conn1.close(); upsertConn.close(); String query = "SELECT a_string, b_string FROM aTable WHERE organization_id=? and a_integer = 5"; - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30)); Connection conn = DriverManager.getConnection(getUrl(), props); try { PreparedStatement statement = conn.prepareStatement(query); @@ -280,7 +282,7 @@ public class QueryIT extends BaseQueryIT { upsertConn.close(); String query = "SELECT organization_id, a_string AS a FROM atable WHERE organization_id=? and a_integer = 5"; - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); Connection conn = DriverManager.getConnection(getUrl(), props); PreparedStatement statement = conn.prepareStatement(query); statement.setString(1, tenantId); @@ -813,15 +815,15 @@ public class QueryIT extends BaseQueryIT { public void testSumOverNullIntegerColumn() throws Exception { String query = "SELECT sum(a_integer) FROM aTable a"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(true); conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) VALUES('" + getOrganizationId() + "','" + ROW3 + "',NULL)"); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30)); Connection conn1 = DriverManager.getConnection(getUrl(), props); analyzeTable(conn1, "ATABLE"); conn1.close(); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 5)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 50)); conn = DriverManager.getConnection(getUrl(), props); try { PreparedStatement statement = conn.prepareStatement(query); @@ -832,15 +834,15 @@ public class QueryIT extends BaseQueryIT { } finally { conn.close(); } - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 7)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70)); conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(true); conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) SELECT organization_id, entity_id, null FROM atable"); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60)); conn1 = DriverManager.getConnection(getUrl(), props); analyzeTable(conn1, "ATABLE"); conn1.close(); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 9)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 90)); conn = DriverManager.getConnection(getUrl(), props); try { PreparedStatement statement = conn.prepareStatement(query); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java index ea59b85..cc08d15 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java @@ -20,7 +20,7 @@ package org.apache.phoenix.end2end; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category(HBaseManagedTimeTest.class) +@Category(NeedsOwnMiniClusterTest.class) public class SaltedViewIT extends BaseViewIT { /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java index 8ef1024..d997e36 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.util.TestUtil.analyzeTable; +import static org.apache.phoenix.util.TestUtil.getAllSplits; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -26,14 +28,16 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.ReadOnlyTableException; import org.apache.phoenix.schema.TableNotFoundException; import org.junit.Test; import org.junit.experimental.categories.Category; -@Category(HBaseManagedTimeTest.class) +@Category(NeedsOwnMiniClusterTest.class) public class ViewIT extends BaseViewIT { @Test @@ -55,6 +59,11 @@ public class ViewIT extends BaseViewIT { } conn.commit(); + analyzeTable(conn, "v"); + + List<KeyRange> splits = getAllSplits(conn, "v"); + assertEquals(4, splits.size()); + int count = 0; ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM v"); while (rs.next()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 7172430..3966953 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -131,6 +131,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver if(ScanUtil.isAnalyzeTable(scan)) { scan.setStartRow(HConstants.EMPTY_START_ROW); scan.setStopRow(HConstants.EMPTY_END_ROW); + scan.setFilter(null); } return s; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java index bc45389..c8c2dba 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java @@ -44,7 +44,6 @@ import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.compile.StatementContext; -import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.filter.ColumnProjectionFilter; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.job.JobManager.JobCallable; @@ -54,15 +53,14 @@ import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.ViewType; -import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.StaleRegionBoundaryCacheException; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SQLCloseables; @@ -89,7 +87,8 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { private static final Logger logger = LoggerFactory.getLogger(ParallelIterators.class); private final List<List<Scan>> scans; private final List<KeyRange> splits; - private final PTable physicalTable; + private final PTableStats tableStats; + private final byte[] physicalTableName; private final QueryPlan plan; private final ParallelIteratorFactory iteratorFactory; @@ -113,10 +112,13 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { this.plan = plan; StatementContext context = plan.getContext(); TableRef tableRef = plan.getTableRef(); + PTable table = tableRef.getTable(); FilterableStatement statement = plan.getStatement(); RowProjector projector = plan.getProjector(); MetaDataClient client = new MetaDataClient(context.getConnection()); - PTable physicalTable = tableRef.getTable(); + physicalTableName = table.getPhysicalName().getBytes(); + tableStats = useStats() ? new MetaDataClient(context.getConnection()).getTableStats(table) : PTableStats.EMPTY_STATS; +/* PTable physicalTable = tableRef.getTable(); String physicalName = tableRef.getTable().getPhysicalName().getString(); if ((physicalTable.getViewIndexId() == null) && (!physicalName.equals(physicalTable.getName().getString()))) { // tableRef is not for the physical table String physicalSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalName); @@ -124,7 +126,7 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { // TODO: this will be an extra RPC to ensure we have the latest guideposts, but is almost always // unnecessary. We should instead track when the last time an update cache was done for this // for physical table and not do it again until some interval has passed (it's ok to use stale stats). - MetaDataMutationResult result = client.updateCache(null, /* use global tenant id to get physical table */ + MetaDataMutationResult result = client.updateCache(null, use global tenant id to get physical table physicalSchemaName, physicalTableName); physicalTable = result.getTable(); if(physicalTable == null) { @@ -133,9 +135,8 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { .getTable(new PTableKey(null, physicalTableName)); } } - this.physicalTable = physicalTable; + this.physicalTable = physicalTable;*/ Scan scan = context.getScan(); - PTable table = tableRef.getTable(); if (projector.isProjectEmptyKeyValue()) { Map<byte [], NavigableSet<byte []>> familyMap = scan.getFamilyMap(); // If nothing projected into scan and we only have one column family, just allow everything @@ -272,6 +273,22 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { return scans; } + private PTable getTable() { + return plan.getTableRef().getTable(); + } + + private boolean useStats() { + Scan scan = context.getScan(); + boolean isPointLookup = context.getScanRanges().isPointLookup(); + /* + * Don't use guide posts if: 1) We're doing a point lookup, as HBase is fast enough at those to not need them to be further + * parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan entire regions worth of data + * to track where to put the guide posts. + */ + if (isPointLookup || ScanUtil.isAnalyzeTable(scan)) { return false; } + return true; + } + private static List<byte[]> toBoundaries(List<HRegionLocation> regionLocations) { int nBoundaries = regionLocations.size() - 1; List<byte[]> ranges = Lists.newArrayListWithExpectedSize(nBoundaries); @@ -298,11 +315,7 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { return guideIndex; } - private List<byte[]> getGuidePosts(PTable table) { - Scan scan = context.getScan(); - boolean isPointLookup = context.getScanRanges().isPointLookup(); - byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(table); - List<byte[]> gps = Collections.emptyList(); + private List<byte[]> getGuidePosts() { /* * Don't use guide posts if: * 1) We're doing a point lookup, as HBase is fast enough at those @@ -310,24 +323,31 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { * 2) We're collecting stats, as in this case we need to scan entire * regions worth of data to track where to put the guide posts. */ - if (!isPointLookup && !ScanUtil.isAnalyzeTable(scan)) { - if (table.getColumnFamilies().isEmpty()) { - // For sure we can get the defaultCF from the table - return table.getGuidePosts(); - } - try { - if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) { - // If default CF is not used in scan, use first CF referenced in scan - return table.getColumnFamily(scan.getFamilyMap().keySet().iterator().next()).getGuidePosts(); - } + if (!useStats()) { + return Collections.emptyList(); + } + + List<byte[]> gps = null; + PTable table = getTable(); + Map<byte[],List<byte[]>> guidePostMap = tableStats.getGuidePosts(); + byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); + if (table.getColumnFamilies().isEmpty()) { + // For sure we can get the defaultCF from the table + gps = guidePostMap.get(defaultCF); + } else { + Scan scan = context.getScan(); + if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) { + // If default CF is not used in scan, use first CF referenced in scan + gps = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next()); + } else { // Otherwise, favor use of default CF. - return table.getColumnFamily(defaultCF).getGuidePosts(); - } catch (ColumnFamilyNotFoundException cfne) { - // Alter table does this + gps = guidePostMap.get(defaultCF); } } + if (gps == null) { + return Collections.emptyList(); + } return gps; - } private static String toString(List<byte[]> gps) { @@ -348,12 +368,13 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { if (scan == null) { return scans; } + PTable table = getTable(); if (!scans.isEmpty()) { boolean startNewScanList = false; if (!plan.isRowKeyOrdered()) { startNewScanList = true; } else if (crossedRegionBoundary) { - if (physicalTable.getBucketNum() != null) { + if (table.getBucketNum() != null) { byte[] previousStartKey = scans.get(scans.size()-1).getStartRow(); byte[] currentStartKey = scan.getStartRow(); byte[] prefix = ScanUtil.getPrefix(previousStartKey, SaltingUtil.NUM_SALTING_BYTES); @@ -377,11 +398,12 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { */ private List<List<Scan>> getParallelScans(final Scan scan) throws SQLException { List<HRegionLocation> regionLocations = context.getConnection().getQueryServices() - .getAllTableRegions(physicalTable.getPhysicalName().getBytes()); + .getAllTableRegions(physicalTableName); List<byte[]> regionBoundaries = toBoundaries(regionLocations); ScanRanges scanRanges = context.getScanRanges(); - boolean isSalted = physicalTable.getBucketNum() != null; - List<byte[]> gps = getGuidePosts(physicalTable); + PTable table = getTable(); + boolean isSalted = table.getBucketNum() != null; + List<byte[]> gps = getGuidePosts(); if (logger.isDebugEnabled()) { logger.debug("Guideposts: " + toString(gps)); } @@ -478,7 +500,7 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { } catch (StaleRegionBoundaryCacheException e2) { // Catch only to try to recover from region boundary cache being out of date List<List<Pair<Scan,Future<PeekingResultIterator>>>> newFutures = Lists.newArrayListWithExpectedSize(2); if (!clearedCache) { // Clear cache once so that we rejigger job based on new boundaries - services.clearTableRegionCache(physicalTable.getName().getBytes()); + services.clearTableRegionCache(physicalTableName); clearedCache = true; } // Resubmit just this portion of work again http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java index 2b4a109..3ae0e81 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java @@ -39,6 +39,7 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceKey; +import org.apache.phoenix.schema.stats.PTableStats; public interface ConnectionQueryServices extends QueryServices, MetaDataMutated { @@ -104,4 +105,9 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated public String getUserName(); public void incrementTableTimeStamp(final byte[] tenantId, final byte[] schemaName, final byte[] tableName, long clientTS) throws SQLException; + + public PTableStats getTableStats(String physicalName); + public void addTableStats(String physicalName, PTableStats tableStats); + + public void clearCache() throws SQLException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 7b8d79c..478ce69 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -36,6 +36,7 @@ import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.GuardedBy; @@ -103,6 +104,7 @@ import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceKey; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; +import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.Closeables; import org.apache.phoenix.util.MetaDataUtil; @@ -116,6 +118,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Throwables; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -124,13 +128,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class); private static final int INITIAL_CHILD_SERVICES_CAPACITY = 100; private static final int DEFAULT_OUT_OF_ORDER_MUTATIONS_WAIT_TIME_MS = 1000; + // Max number of cached table stats for view or shared index physical tables + private static final int MAX_TABLE_STATS_CACHE_ENTRIES = 512; protected final Configuration config; // Copy of config.getProps(), but read-only to prevent synchronization that we // don't need. private final ReadOnlyProps props; private final String userName; private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices; - + private final Cache<String, PTableStats> tableStatsCache; // Cache the latest meta data here for future connections // writes guarded by "latestMetaDataLock" private volatile PMetaData latestMetaData; @@ -189,6 +195,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // find the HBase version and use that to determine the KeyValueBuilder that should be used String hbaseVersion = VersionInfo.getVersion(); this.kvBuilder = KeyValueBuilder.get(hbaseVersion); + long halfStatsUpdateFreq = config.getLong( + QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2; + tableStatsCache = CacheBuilder.newBuilder() + .maximumSize(MAX_TABLE_STATS_CACHE_ENTRIES) + .expireAfterWrite(halfStatsUpdateFreq, TimeUnit.MILLISECONDS) + .build(); } private void openConnection() throws SQLException { @@ -281,6 +294,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } finally { try { + tableStatsCache.invalidateAll(); super.close(); } catch (SQLException e) { if (sqlE == null) { @@ -1035,6 +1049,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (dropMetadata) { dropTables(result.getTableNamesToDelete()); } + invalidateTables(result.getTableNamesToDelete()); if (tableType == PTableType.TABLE) { byte[] physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes); long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData); @@ -1047,6 +1062,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement return result; } + private void invalidateTables(final List<byte[]> tableNamesToDelete) { + if (tableNamesToDelete != null) { + for (byte[] tableName : tableNamesToDelete) { + tableStatsCache.invalidate(Bytes.toString(tableName)); + } + } + } + private void dropTables(final List<byte[]> tableNamesToDelete) throws SQLException { HBaseAdmin admin = null; SQLException sqlE = null; @@ -1224,6 +1247,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (dropMetadata) { dropTables(result.getTableNamesToDelete()); } + invalidateTables(result.getTableNamesToDelete()); break; default: break; @@ -1388,7 +1412,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement * Clears the Phoenix meta data cache on each region server * @throws SQLException */ - protected void clearCache() throws SQLException { + @Override + public void clearCache() throws SQLException { try { SQLException sqlE = null; HTableInterface htable = this.getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES); @@ -1407,6 +1432,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement sqlE = new SQLException(e); } finally { try { + tableStatsCache.invalidateAll(); htable.close(); } catch (IOException e) { if (sqlE == null) { @@ -1829,6 +1855,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement sqlE = new SQLException(e); } finally { try { + if (tenantId.length == 0) + tableStatsCache.invalidate(SchemaUtil.getTableName(schemaName, tableName)); htable.close(); } catch (IOException e) { if (sqlE == null) { @@ -1844,4 +1872,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement throw new SQLException(ServerUtil.parseServerException(e)); } } + @Override + public PTableStats getTableStats(String physicalName) { + return tableStatsCache.getIfPresent(physicalName); + } + + @Override + public void addTableStats(String physicalName, PTableStats tableStats) { + tableStatsCache.put(physicalName, tableStats); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 3e4643f..dd0cf54 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -65,6 +65,7 @@ import org.apache.phoenix.schema.SequenceKey; import org.apache.phoenix.schema.SequenceNotFoundException; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; +import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; @@ -413,4 +414,17 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple public String getUserName() { return userName; } + + @Override + public PTableStats getTableStats(String physicalName) { + return PTableStats.EMPTY_STATS; + } + + @Override + public void addTableStats(String physicalName, PTableStats tableStats) { + } + + @Override + public void clearCache() throws SQLException { + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java index 86523fd..2bcacc6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java @@ -41,6 +41,7 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceKey; +import org.apache.phoenix.schema.stats.PTableStats; public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices { @@ -232,4 +233,19 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple throws SQLException { getDelegate().incrementTableTimeStamp(tenantId, schemaName, tableName, clientTS); } + + @Override + public PTableStats getTableStats(String physicalName) { + return getDelegate().getTableStats(physicalName); + } + + @Override + public void addTableStats(String physicalName, PTableStats tableStats) { + getDelegate().addTableStats(physicalName, tableStats); + } + + @Override + public void clearCache() throws SQLException { + getDelegate().clearCache(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 3a0fa63..ecf4818 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -123,6 +123,7 @@ public interface QueryServices extends SQLCloseable { public static final String STATS_UPDATE_FREQ_MS_ATTRIB = "phoenix.stats.updateFrequency"; public static final String MIN_STATS_UPDATE_FREQ_MS_ATTRIB = "phoenix.stats.minUpdateFrequency"; public static final String STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB = "phoenix.stats.guidepost.width"; + public static final String STATS_GUIDEPOST_PER_REGION_ATTRIB = "phoenix.stats.guidepost.per.region"; /** * Get executor service used for parallel scans http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 467aae1..5fdf165 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -130,6 +130,7 @@ public class QueryServicesOptions { public static final long DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE = 1024 * 1024 * 30; public static final int DEFAULT_STATS_UPDATE_FREQ_MS = 15 * 60000; // 15min + public static final int DEFAULT_GUIDE_POSTS_PER_REGION = 20; public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = DEFAULT_STATS_UPDATE_FREQ_MS/2; private final Configuration config; @@ -184,7 +185,6 @@ public class QueryServicesOptions { .setIfUnset(GROUPBY_SPILL_FILES_ATTRIB, DEFAULT_GROUPBY_SPILL_FILES) .setIfUnset(SEQUENCE_CACHE_SIZE_ATTRIB, DEFAULT_SEQUENCE_CACHE_SIZE) .setIfUnset(SCAN_RESULT_CHUNK_SIZE, DEFAULT_SCAN_RESULT_CHUNK_SIZE) - .setIfUnset(STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE); ; // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 8bb91b4..b0549d2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; @@ -130,10 +131,13 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTable.ViewType; +import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.StatisticsUtil; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -292,13 +296,19 @@ public class MetaDataClient { public MetaDataMutationResult updateCache(String schemaName, String tableName) throws SQLException { return updateCache(schemaName, tableName, false); } - + + private long getClientTimeStamp() { + Long scn = connection.getSCN(); + long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn; + return clientTimeStamp; + } + private MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName, boolean alwaysHitServer) throws SQLException { + long clientTimeStamp = getClientTimeStamp(); Long scn = connection.getSCN(); boolean systemTable = SYSTEM_CATALOG_SCHEMA.equals(schemaName); // System tables must always have a null tenantId tenantId = systemTable ? null : tenantId; - long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn; PTable table = null; String fullTableName = SchemaUtil.getTableName(schemaName, tableName); long tableTimestamp = HConstants.LATEST_TIMESTAMP; @@ -474,11 +484,11 @@ public class MetaDataClient { public MutationState updateStatistics(UpdateStatisticsStatement updateStatisticsStmt) throws SQLException { // Check before updating the stats if we have reached the configured time to reupdate the stats once again - final long msMinBetweenUpdates = connection - .getQueryServices() - .getProps() + ReadOnlyProps props = connection.getQueryServices().getProps(); + final long msMinBetweenUpdates = props .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, - QueryServicesOptions.DEFAULT_MIN_STATS_UPDATE_FREQ_MS); + props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2); ColumnResolver resolver = FromCompiler.getResolver(updateStatisticsStmt, connection); PTable table = resolver.getTables().get(0).getTable(); List<PTable> indexes = table.getIndexes(); @@ -2194,4 +2204,59 @@ public class MetaDataClient { .build().buildException(); } } + + public PTableStats getTableStats(PTable table) throws SQLException { + boolean isView = table.getType() == PTableType.VIEW; + boolean isSharedIndex = table.getViewIndexId() != null; + if (!isView && !isSharedIndex) { + return table.getTableStats(); + } + String physicalName = table.getPhysicalName().getString(); + // If we have a VIEW or a local or view INDEX, check our cache rather + // than updating the cache for that table to prevent an extra roundtrip. + PTableStats tableStats = connection.getQueryServices().getTableStats(physicalName); + if (tableStats != null) { + return tableStats; + } + if (isView) { + String physicalSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalName); + String physicalTableName = SchemaUtil.getTableNameFromFullName(physicalName); + MetaDataMutationResult result = updateCache(null, /* use global tenant id to get physical table */ + physicalSchemaName, physicalTableName); + PTable physicalTable = result.getTable(); + if(physicalTable == null) { + // We should be able to find the physical table, as we found the logical one + // Might mean the physical table as just deleted. + logger.warn("Unable to retrieve physical table " + physicalName + " for table " + table.getName().getString()); + throw new TableNotFoundException(table.getSchemaName().getString(),table.getTableName().getString()); + } + tableStats = physicalTable.getTableStats(); + } else { + /* + * Otherwise, we have a shared view. This case is tricky, because we don't have + * table metadata for it, only an HBase table. We do have stats, though, so we'll + * query them directly here and cache them so we don't keep querying for them. + */ + HTableInterface statsHTable = connection.getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); + try { + long clientTimeStamp = getClientTimeStamp(); + tableStats = StatisticsUtil.readStatistics(statsHTable, table.getPhysicalName().getBytes(), clientTimeStamp); + } catch (IOException e) { + logger.warn("Unable to read from stats table", e); + // Just cache empty stats. We'll try again after some time anyway. + tableStats = PTableStats.EMPTY_STATS; + } finally { + try { + statsHTable.close(); + } catch (IOException e) { + // Log, but continue. We have our stats anyway now. + logger.warn("Unable to close stats table", e); + } + } + } + // Cache these stats so that we don't keep making a roundrip just to get the stats (as + // they don't change very often. + connection.getQueryServices().addTableStats(physicalName, tableStats); + return tableStats; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java.orig ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java.orig b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java.orig index 9b744e1..8bb91b4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java.orig +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java.orig @@ -36,11 +36,14 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIME import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LAST_STATS_UPDATE_TIME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.REGION_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; @@ -59,8 +62,10 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADAT import static org.apache.phoenix.schema.PDataType.VARCHAR; import java.io.IOException; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; @@ -79,9 +84,12 @@ import java.util.Set; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -90,6 +98,8 @@ import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.MutationPlan; import org.apache.phoenix.compile.PostDDLCompiler; import org.apache.phoenix.compile.PostIndexDDLCompiler; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode; @@ -98,6 +108,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.MutationState; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.AddColumnStatement; import org.apache.phoenix.parse.AlterIndexStatement; import org.apache.phoenix.parse.ColumnDef; @@ -113,6 +124,7 @@ import org.apache.phoenix.parse.NamedTableNode; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.PrimaryKeyConstraint; import org.apache.phoenix.parse.TableName; +import org.apache.phoenix.parse.UpdateStatisticsStatement; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -260,7 +272,16 @@ public class MetaDataClient { MetaDataMutationResult result = updateCache(schemaName, tableName, true); return result.getMutationTime(); } - + + private MetaDataMutationResult updateCache(String schemaName, String tableName, boolean alwaysHitServer) + throws SQLException { + return updateCache(connection.getTenantId(), schemaName, tableName, alwaysHitServer); + } + + public MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName) throws SQLException { + return updateCache(tenantId, schemaName, tableName, false); + } + /** * Update the cache with the latest as of the connection scn. * @param schemaName @@ -272,11 +293,11 @@ public class MetaDataClient { return updateCache(schemaName, tableName, false); } - private MetaDataMutationResult updateCache(String schemaName, String tableName, boolean alwaysHitServer) throws SQLException { // TODO: pass byte[] here + private MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName, boolean alwaysHitServer) throws SQLException { Long scn = connection.getSCN(); boolean systemTable = SYSTEM_CATALOG_SCHEMA.equals(schemaName); // System tables must always have a null tenantId - PName tenantId = systemTable ? null : connection.getTenantId(); + tenantId = systemTable ? null : tenantId; long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn; PTable table = null; String fullTableName = SchemaUtil.getTableName(schemaName, tableName); @@ -450,6 +471,72 @@ public class MetaDataClient { return connection.getQueryServices().updateData(plan); } + public MutationState updateStatistics(UpdateStatisticsStatement updateStatisticsStmt) + throws SQLException { + // Check before updating the stats if we have reached the configured time to reupdate the stats once again + final long msMinBetweenUpdates = connection + .getQueryServices() + .getProps() + .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, + QueryServicesOptions.DEFAULT_MIN_STATS_UPDATE_FREQ_MS); + ColumnResolver resolver = FromCompiler.getResolver(updateStatisticsStmt, connection); + PTable table = resolver.getTables().get(0).getTable(); + List<PTable> indexes = table.getIndexes(); + List<PTable> tables = Lists.newArrayListWithExpectedSize(1 + indexes.size()); + if (updateStatisticsStmt.updateColumns()) { + tables.add(table); + } + if (updateStatisticsStmt.updateIndex()) { + tables.addAll(indexes); + } + for(PTable pTable : tables) { + updateStatisticsInternal(msMinBetweenUpdates, pTable); + } + return new MutationState(1, connection); + } + + private MutationState updateStatisticsInternal(long msMinBetweenUpdates, PTable table) throws SQLException { + PName physicalName = table.getPhysicalName(); + byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY; + Long scn = connection.getSCN(); + // Always invalidate the cache + long clientTS = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn; + String query = "SELECT CURRENT_DATE() - " + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + + " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY + + " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL"; + ResultSet rs = connection.createStatement().executeQuery(query); + long msSinceLastUpdate = Long.MAX_VALUE; + if (rs.next()) { + msSinceLastUpdate = rs.getLong(1); + } + if (msSinceLastUpdate >= msMinBetweenUpdates) { + // Here create the select query. + String countQuery = "SELECT /*+ NO_CACHE NO_INDEX */ count(*) FROM " + table.getName().getString(); + PhoenixStatement statement = (PhoenixStatement) connection.createStatement(); + QueryPlan plan = statement.compileQuery(countQuery); + Scan scan = plan.getContext().getScan(); + // Add all CF in the table + scan.getFamilyMap().clear(); + for (PColumnFamily family : table.getColumnFamilies()) { + scan.addFamily(family.getName().getBytes()); + } + scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES); + KeyValue kv = plan.iterator().next().getValue(0); + ImmutableBytesWritable tempPtr = plan.getContext().getTempPtr(); + tempPtr.set(kv.getValue()); + // A single Cell will be returned with the count(*) - we decode that here + long rowCount = PDataType.LONG.getCodec().decodeLong(tempPtr, SortOrder.getDefault()); + // We need to update the stats table so that client will pull the new one with + // the updated stats. + connection.getQueryServices().incrementTableTimeStamp(tenantIdBytes, + Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())), + Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTS); + return new MutationState(0, connection, rowCount); + } else { + return new MutationState(0, connection); + } + } + private MutationState buildIndexAtTimeStamp(PTable index, NamedTableNode dataTableNode) throws SQLException { // If our connection is at a fixed point-in-time, we need to open a new // connection so that our new index table is visible. @@ -792,7 +879,6 @@ public class MetaDataClient { String parentTableName = null; PName tenantId = connection.getTenantId(); String tenantIdStr = tenantId == null ? null : connection.getTenantId().getString(); - boolean isParentImmutableRows = false; boolean multiTenant = false; Integer saltBucketNum = null; String defaultFamilyName = null; @@ -814,7 +900,6 @@ public class MetaDataClient { } multiTenant = parent.isMultiTenant(); - isParentImmutableRows = parent.isImmutableRows(); parentTableName = parent.getTableName().getString(); // Pass through data table sequence number so we can check it hasn't changed PreparedStatement incrementStatement = connection.prepareStatement(INCREMENT_SEQ_NUM); @@ -905,9 +990,7 @@ public class MetaDataClient { boolean disableWAL = false; Boolean disableWALProp = (Boolean) tableProps.remove(PhoenixDatabaseMetaData.DISABLE_WAL); - if (disableWALProp == null) { - disableWAL = isParentImmutableRows; // By default, disable WAL for immutable indexes - } else { + if (disableWALProp != null) { disableWAL = disableWALProp; } // Delay this check as it is supported to have IMMUTABLE_ROWS and SALT_BUCKETS defined on views @@ -1112,6 +1195,7 @@ public class MetaDataClient { // Bootstrapping for our SYSTEM.TABLE that creates itself before it exists if (SchemaUtil.isMetaTable(schemaName,tableName)) { + // TODO: what about stats for system catalog? PTable table = PTableImpl.makePTable(tenantId,PNameFactory.newName(schemaName), PNameFactory.newName(tableName), tableType, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, PNameFactory.newName(QueryConstants.SYSTEM_TABLE_PK_NAME), null, columns, null, Collections.<PTable>emptyList(), @@ -1320,7 +1404,8 @@ public class MetaDataClient { return dropTable(schemaName, tableName, parentTableName, PTableType.INDEX, statement.ifExists(), false); } - private MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType, boolean ifExists, boolean cascade) throws SQLException { + private MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType, + boolean ifExists, boolean cascade) throws SQLException { connection.rollback(); boolean wasAutoCommit = connection.getAutoCommit(); try { @@ -1378,13 +1463,14 @@ public class MetaDataClient { tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); } } + // Delete everything in the column. You'll still be able to do queries at earlier timestamps + tableRefs.add(new TableRef(null, table, ts, false)); + // TODO: Let the standard mutable secondary index maintenance handle this? + for (PTable index: table.getIndexes()) { + tableRefs.add(new TableRef(null, index, ts, false)); + } + deleteFromStatsTable(tableRefs, ts); if (!dropMetaData) { - // Delete everything in the column. You'll still be able to do queries at earlier timestamps - tableRefs.add(new TableRef(null, table, ts, false)); - // TODO: Let the standard mutable secondary index maintenance handle this? - for (PTable index: table.getIndexes()) { - tableRefs.add(new TableRef(null, index, ts, false)); - } MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections.<PColumn>emptyList(), ts); return connection.getQueryServices().updateData(plan); } @@ -1396,6 +1482,41 @@ public class MetaDataClient { connection.setAutoCommit(wasAutoCommit); } } + + private void deleteFromStatsTable(List<TableRef> tableRefs, long ts) throws SQLException { + Properties props = new Properties(connection.getClientInfo()); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + Connection conn = DriverManager.getConnection(connection.getURL(), props); + conn.setAutoCommit(true); + boolean success = false; + SQLException sqlException = null; + try { + StringBuilder buf = new StringBuilder("DELETE FROM SYSTEM.STATS WHERE PHYSICAL_NAME IN ("); + for (TableRef ref : tableRefs) { + buf.append("'" + ref.getTable().getName().getString() + "',"); + } + buf.setCharAt(buf.length() - 1, ')'); + conn.createStatement().execute(buf.toString()); + success = true; + } catch (SQLException e) { + sqlException = e; + } finally { + try { + conn.close(); + } catch (SQLException e) { + if (sqlException == null) { + // If we're not in the middle of throwing another exception + // then throw the exception we got on close. + if (success) { + sqlException = e; + } + } else { + sqlException.setNextException(e); + } + } + if (sqlException != null) { throw sqlException; } + } + } private MutationCode processMutationResult(String schemaName, String tableName, MetaDataMutationResult result) throws SQLException { final MutationCode mutationCode = result.getMutationCode(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java index 01c236f..c5fe9b1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java @@ -18,7 +18,6 @@ package org.apache.phoenix.schema; import java.util.Collection; -import java.util.List; /** * @@ -53,5 +52,4 @@ public interface PColumnFamily { int getEstimatedSize(); - List<byte[]> getGuidePosts(); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java index 9841233..2e29656 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java @@ -17,7 +17,6 @@ */ package org.apache.phoenix.schema; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -35,7 +34,6 @@ public class PColumnFamilyImpl implements PColumnFamily { private final Map<String, PColumn> columnByString; private final Map<byte[], PColumn> columnByBytes; private final int estimatedSize; - private List<byte[]> guidePosts = Collections.emptyList(); @Override public int getEstimatedSize() { @@ -43,22 +41,10 @@ public class PColumnFamilyImpl implements PColumnFamily { } public PColumnFamilyImpl(PName name, List<PColumn> columns) { - this(name, columns, null); - } - - public PColumnFamilyImpl(PName name, List<PColumn> columns, List<byte[]> guidePosts) { Preconditions.checkNotNull(name); // Include guidePosts also in estimating the size long estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.POINTER_SIZE * 5 + SizedUtil.INT_SIZE + name.getEstimatedSize() + SizedUtil.sizeOfMap(columns.size()) * 2 + SizedUtil.sizeOfArrayList(columns.size()); - if(guidePosts != null) { - int guidePostsSize = guidePosts.size(); - estimatedSize += SizedUtil.sizeOfArrayList(guidePostsSize); - for(byte[] gps : guidePosts) { - estimatedSize += gps.length; - } - this.guidePosts = guidePosts; - } this.name = name; this.columns = ImmutableList.copyOf(columns); ImmutableMap.Builder<String, PColumn> columnByStringBuilder = ImmutableMap.builder(); @@ -100,9 +86,4 @@ public class PColumnFamilyImpl implements PColumnFamily { } return column; } - - @Override - public List<byte[]> getGuidePosts() { - return guidePosts; - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index 2d11c22..a766d1e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -217,13 +217,6 @@ public interface PTable extends Writable { */ int newKey(ImmutableBytesWritable key, byte[][] values); - /** - * Return the statistics table associated with this PTable. A list of - * guide posts are return - * @return the statistics table. - */ - List<byte[]> getGuidePosts(); - RowKeySchema getRowKeySchema(); /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 0bcd7a8..007abd2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -111,7 +111,6 @@ public class PTableImpl implements PTable { private ViewType viewType; private Short viewIndexId; private int estimatedSize; - private List<byte[]> guidePosts = Collections.emptyList(); private PTableStats tableStats = PTableStats.EMPTY_STATS; public PTableImpl() { @@ -353,24 +352,13 @@ public class PTableImpl implements PTable { estimatedSize += rowKeySchema.getEstimatedSize(); Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator(); PColumnFamily[] families = new PColumnFamily[familyMap.size()]; - if (families.length == 0) { - byte[] defaultFamilyNameBytes = (defaultFamilyName == null) ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : defaultFamilyName.getBytes(); - List<byte[]> guidePosts = stats.getGuidePosts().get(defaultFamilyNameBytes); - if (guidePosts != null) { - this.guidePosts = guidePosts; - estimatedSize += SizedUtil.sizeOfArrayList(guidePosts.size()); - for (byte[] gps : guidePosts) { - estimatedSize += gps.length; - } - } - } + estimatedSize += this.tableStats.getEstimatedSize(); ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder(); ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap .orderedBy(Bytes.BYTES_COMPARATOR); for (int i = 0; i < families.length; i++) { Map.Entry<PName,List<PColumn>> entry = iterator.next(); - List<byte[]> famGuidePosts = stats.getGuidePosts().get(entry.getKey().getBytes()); - PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue(), famGuidePosts); + PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue()); families[i] = family; familyByString.put(family.getName().getString(), family); familyByBytes.put(family.getName().getBytes(), family); @@ -719,11 +707,6 @@ public class PTableImpl implements PTable { } @Override - public List<byte[]> getGuidePosts() { - return guidePosts; - } - - @Override public void readFields(DataInput input) throws IOException { byte[] tenantIdBytes = Bytes.readByteArray(input); byte[] schemaNameBytes = Bytes.readByteArray(input); http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java index d0f9ecb..d902cb9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java @@ -53,6 +53,11 @@ public interface PTableStats extends Writable { public void readFields(DataInput arg0) throws IOException { throw new UnsupportedOperationException(); } + + @Override + public int getEstimatedSize() { + return 0; + } }; /** @@ -61,4 +66,6 @@ public interface PTableStats extends Writable { * @return */ SortedMap<byte[], List<byte[]>> getGuidePosts(); + + int getEstimatedSize(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index c3e1803..159fb79 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -21,12 +21,14 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.util.SizedUtil; import com.google.common.collect.Lists; import com.sun.istack.NotNull; @@ -37,6 +39,7 @@ import com.sun.istack.NotNull; */ public class PTableStatsImpl implements PTableStats { private final SortedMap<byte[], List<byte[]>> guidePosts; + private final int estimatedSize; public PTableStatsImpl() { this(new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR)); @@ -44,6 +47,17 @@ public class PTableStatsImpl implements PTableStats { public PTableStatsImpl(@NotNull SortedMap<byte[], List<byte[]>> guidePosts) { this.guidePosts = guidePosts; + int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size()); + for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) { + byte[] cf = entry.getKey(); + estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; + List<byte[]> keys = entry.getValue(); + estimatedSize += SizedUtil.sizeOfArrayList(keys.size()); + for (byte[] key : keys) { + estimatedSize += SizedUtil.ARRAY_SIZE + key.length; + } + } + this.estimatedSize = estimatedSize; } @Override @@ -77,4 +91,31 @@ public class PTableStatsImpl implements PTableStats { guidePosts.put(key, value); } } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("PTableStats ["); + for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) { + buf.append(Bytes.toStringBinary(entry.getKey())); + buf.append(":("); + List<byte[]> keys = entry.getValue(); + if (!keys.isEmpty()) { + for (byte[] key : keys) { + buf.append(Bytes.toStringBinary(key)); + buf.append(","); + } + buf.setLength(buf.length()-1); + } + buf.append(")"); + } + buf.append("]"); + return buf.toString(); + } + + @Override + public int getEstimatedSize() { + return estimatedSize; + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index a5eb387..29706c7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -59,15 +59,19 @@ public class StatisticsCollector { // Ensures that either analyze or compaction happens at any point of time. private static final Log LOG = LogFactory.getLog(StatisticsCollector.class); - public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { + public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) + throws IOException { + Configuration config = env.getConfiguration(); + HTableInterface statsHTable = env.getTable((PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES)); guidepostDepth = - env.getConfiguration().getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE); + config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, + statsHTable.getTableDescriptor().getMaxFileSize() / + config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, + QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION)); // Get the stats table associated with the current table on which the CP is // triggered - HTableInterface statsHTable = env.getTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - this.statsTable = StatisticsWriter.getStatisticsTable(statsHTable, tableName, clientTimeStamp); + this.statsTable = StatisticsWriter.newWriter(statsHTable, tableName, clientTimeStamp); } public void close() throws IOException { http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 60e607c..5bf39c5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -38,7 +38,8 @@ import org.apache.phoenix.util.TimeKeeper; * Wrapper to access the statistics table SYSTEM.STATS using the HTable. */ public class StatisticsWriter implements Closeable { - public static StatisticsWriter getStatisticsTable(HTableInterface hTable, String tableName, long clientTimeStamp) throws IOException { + public static StatisticsWriter newWriter(HTableInterface hTable, String tableName, long clientTimeStamp) + throws IOException { if (clientTimeStamp == HConstants.LATEST_TIMESTAMP) { clientTimeStamp = TimeKeeper.SYSTEM.getCurrentTime(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java index c49b0e7..f82c1b8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java @@ -38,6 +38,7 @@ public class SizedUtil { public static final int INT_SIZE = 4; public static final int LONG_SIZE = 8; + public static final int TREE_MAP_SIZE = OBJECT_SIZE + INT_SIZE * 2 + POINTER_SIZE * 2; public static final int MAP_ENTRY_SIZE = OBJECT_SIZE + 3 * POINTER_SIZE + INT_SIZE; public static final int IMMUTABLE_BYTES_WRITABLE_SIZE = OBJECT_SIZE + INT_SIZE * 2 + ARRAY_SIZE; public static final int IMMUTABLE_BYTES_PTR_SIZE = IMMUTABLE_BYTES_WRITABLE_SIZE + INT_SIZE;// Extra is an int field which caches hashcode. @@ -52,6 +53,10 @@ public class SizedUtil { private SizedUtil() { } + public static int sizeOfTreeMap(int size) { + return TREE_MAP_SIZE + (OBJECT_SIZE + INT_SIZE + POINTER_SIZE * 5) * size; + } + public static int sizeOfArrayList(int capacity) { return SizedUtil.OBJECT_SIZE + SizedUtil.POINTER_SIZE + SizedUtil.INT_SIZE + SizedUtil.ARRAY_SIZE + SizedUtil.POINTER_SIZE * capacity; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b79722/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java index e4295ff..16845e8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java @@ -52,7 +52,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl { public static final long DEFAULT_MAX_SERVER_METADATA_CACHE_SIZE = 1024L*1024L*4L; // 4 Mb public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb - public static final long DEFAULT_STATS_HISTOGRAM_DEPTH_BYTES = 2000; public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0; public QueryServicesTestImpl(ReadOnlyProps defaultProps) { @@ -62,7 +61,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl { private static QueryServicesOptions getDefaultServicesOptions() { return withDefaults() .setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS) - .setStatsHistogramDepthBytes(DEFAULT_STATS_HISTOGRAM_DEPTH_BYTES) .setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE) .setQueueSize(DEFAULT_QUEUE_SIZE) .setMaxMemoryPerc(DEFAULT_MAX_MEMORY_PERC)
