PHOENIX-1496 Further reduce work in StatsCollector.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/6cec3465 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/6cec3465 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/6cec3465 Branch: refs/heads/3.0 Commit: 6cec3465bd7712f1965264bc4918e6d4725c4168 Parents: 9313d10 Author: Lars Hofhansl <[email protected]> Authored: Tue Dec 2 10:03:01 2014 -0800 Committer: Lars Hofhansl <[email protected]> Committed: Tue Dec 2 10:11:16 2014 -0800 ---------------------------------------------------------------------- .../UngroupedAggregateRegionObserver.java | 4 +- .../schema/stats/StatisticsCollector.java | 40 ++++++++++++++------ .../phoenix/schema/stats/StatisticsScanner.java | 1 - 3 files changed, 31 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cec3465/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 4cf816a..d561f30 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -438,7 +438,9 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver // when background tasks are updating stats. Instead we track the max timestamp of // the cells and use that. long clientTimeStamp = useCurrentTime ? TimeKeeper.SYSTEM.getCurrentTime() : StatisticsCollector.NO_TIMESTAMP; - StatisticsCollector stats = new StatisticsCollector(c.getEnvironment(), table, clientTimeStamp); + StatisticsCollector stats = new StatisticsCollector( + c.getEnvironment(), table, + clientTimeStamp, store.getFamily().getName()); internalScanner = stats.createCompactionScanner(c.getEnvironment().getRegion(), store, scanner); } catch (IOException e) { // If we can't reach the stats table, don't interrupt the normal http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cec3465/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 7ad1d9b..beaac2f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -54,9 +54,13 @@ public class StatisticsCollector { // Tracks the bytecount per family if it has reached the guidePostsDepth private Map<ImmutableBytesPtr, Boolean> familyMap = Maps.newHashMap(); protected StatisticsWriter statsTable; + private Pair<Long,GuidePostsInfo> cachedGps = null; - public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) - throws IOException { + public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { + this(env, tableName, clientTimeStamp, null); + } + + public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family) throws IOException { Configuration config = env.getConfiguration(); int guidepostPerRegion = config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION); @@ -66,6 +70,13 @@ public class StatisticsCollector { // Get the stats table associated with the current table on which the CP is // triggered this.statsTable = StatisticsWriter.newWriter(env, tableName, clientTimeStamp); + // in a compaction we know the one family ahead of time + if (family != null) { + ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); + familyMap.put(cfKey, true); + cachedGps = new Pair<Long,GuidePostsInfo>(0L,new GuidePostsInfo(0, Collections.<byte[]>emptyList())); + guidePostsMap.put(cfKey, cachedGps); + } } public long getMaxTimeStamp() { @@ -130,8 +141,7 @@ public class StatisticsCollector { if (logger.isDebugEnabled()) { logger.debug("Compaction scanner created for stats"); } - // FIXME: no way to get cf as byte[] ? - ImmutableBytesPtr cfKey = new ImmutableBytesPtr(Bytes.toBytes(store.getColumnFamilyName())); + ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName()); return getInternalScanner(region, store, s, cfKey); } @@ -176,15 +186,21 @@ public class StatisticsCollector { } public void updateStatistic(KeyValue kv) { - ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()); - familyMap.put(cfKey, true); - maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - // TODO : This can be moved to an interface so that we could collect guide posts in different ways - Pair<Long,GuidePostsInfo> gps = guidePostsMap.get(cfKey); - if (gps == null) { - gps = new Pair<Long,GuidePostsInfo>(0L,new GuidePostsInfo(0, Collections.<byte[]>emptyList())); - guidePostsMap.put(cfKey, gps); + + Pair<Long,GuidePostsInfo> gps; + if (cachedGps == null) { + ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength()); + familyMap.put(cfKey, true); + + // TODO : This can be moved to an interface so that we could collect guide posts in different ways + gps = guidePostsMap.get(cfKey); + if (gps == null) { + gps = new Pair<Long,GuidePostsInfo>(0L,new GuidePostsInfo(0, Collections.<byte[]>emptyList())); + guidePostsMap.put(cfKey, gps); + } + } else { + gps = cachedGps; } int kvLength = kv.getLength(); long byteCount = gps.getFirst() + kvLength; http://git-wip-us.apache.org/repos/asf/phoenix/blob/6cec3465/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index 7c76d5f..a58fd75 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -38,7 +38,6 @@ public class StatisticsScanner implements InternalScanner { this.delegate = delegate; this.region = region; this.family = family; - this.tracker.clear(); } @Override
