Repository: phoenix Updated Branches: refs/heads/3.0 a5b797225 -> 75484fb32
Phoenix-1333 Store statistics guideposts as VARBINARY (Ramkrishna S Vasudevan) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/75484fb3 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/75484fb3 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/75484fb3 Branch: refs/heads/3.0 Commit: 75484fb32ae73954e63f1364cb6652760fefe579 Parents: a5b7972 Author: Ramkrishna <ramkrishna.s.vasude...@intel.com> Authored: Mon Oct 13 11:45:44 2014 +0530 Committer: Ramkrishna <ramkrishna.s.vasude...@intel.com> Committed: Mon Oct 13 11:45:44 2014 +0530 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/QueryIT.java | 2 +- .../phoenix/iterate/ParallelIterators.java | 35 ++++-------- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 4 ++ .../apache/phoenix/query/QueryConstants.java | 6 +- .../org/apache/phoenix/schema/PTableImpl.java | 6 +- .../phoenix/schema/stats/PTableStats.java | 4 +- .../phoenix/schema/stats/PTableStatsImpl.java | 25 ++++---- .../schema/stats/StatisticsCollector.java | 60 +++++++++----------- .../phoenix/schema/stats/StatisticsUtil.java | 31 ++-------- .../phoenix/schema/stats/StatisticsWriter.java | 9 ++- 10 files changed, 80 insertions(+), 102 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java index ac1c61b..7282f39 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java @@ -282,7 +282,7 @@ public class QueryIT extends BaseQueryIT { upsertConn.close(); String query = "SELECT organization_id, a_string AS a FROM atable WHERE organization_id=? and a_integer = 5"; - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); Connection conn = DriverManager.getConnection(getUrl(), props); PreparedStatement statement = conn.prepareStatement(query); statement.setString(1, tenantId); http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java index c8c2dba..e4fb0ed 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java @@ -60,6 +60,7 @@ import org.apache.phoenix.schema.PTable.ViewType; import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.StaleRegionBoundaryCacheException; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -115,27 +116,8 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { PTable table = tableRef.getTable(); FilterableStatement statement = plan.getStatement(); RowProjector projector = plan.getProjector(); - MetaDataClient client = new MetaDataClient(context.getConnection()); physicalTableName = table.getPhysicalName().getBytes(); tableStats = useStats() ? new MetaDataClient(context.getConnection()).getTableStats(table) : PTableStats.EMPTY_STATS; -/* PTable physicalTable = tableRef.getTable(); - String physicalName = tableRef.getTable().getPhysicalName().getString(); - if ((physicalTable.getViewIndexId() == null) && (!physicalName.equals(physicalTable.getName().getString()))) { // tableRef is not for the physical table - String physicalSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalName); - String physicalTableName = SchemaUtil.getTableNameFromFullName(physicalName); - // TODO: this will be an extra RPC to ensure we have the latest guideposts, but is almost always - // unnecessary. We should instead track when the last time an update cache was done for this - // for physical table and not do it again until some interval has passed (it's ok to use stale stats). - MetaDataMutationResult result = client.updateCache(null, use global tenant id to get physical table - physicalSchemaName, physicalTableName); - physicalTable = result.getTable(); - if(physicalTable == null) { - client = new MetaDataClient(context.getConnection()); - physicalTable = client.getConnection().getMetaDataCache() - .getTable(new PTableKey(null, physicalTableName)); - } - } - this.physicalTable = physicalTable;*/ Scan scan = context.getScan(); if (projector.isProjectEmptyKeyValue()) { Map<byte [], NavigableSet<byte []>> familyMap = scan.getFamilyMap(); @@ -329,19 +311,26 @@ public class ParallelIterators extends ExplainTable implements ResultIterators { List<byte[]> gps = null; PTable table = getTable(); - Map<byte[],List<byte[]>> guidePostMap = tableStats.getGuidePosts(); + Map<byte[],GuidePostsInfo> guidePostMap = tableStats.getGuidePosts(); byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); if (table.getColumnFamilies().isEmpty()) { // For sure we can get the defaultCF from the table - gps = guidePostMap.get(defaultCF); + if (guidePostMap.get(defaultCF) != null) { + gps = guidePostMap.get(defaultCF).getGuidePosts(); + } } else { Scan scan = context.getScan(); if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) { // If default CF is not used in scan, use first CF referenced in scan - gps = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next()); + GuidePostsInfo guidePostsInfo = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next()); + if (guidePostsInfo != null) { + gps = guidePostsInfo.getGuidePosts(); + } } else { // Otherwise, favor use of default CF. - gps = guidePostMap.get(defaultCF); + if (guidePostMap.get(defaultCF) != null) { + gps = guidePostMap.get(defaultCF).getGuidePosts(); + } } } if (gps == null) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index ad92c3b..84392c0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -231,6 +231,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho public static final byte[] REGION_NAME_BYTES = Bytes.toBytes(REGION_NAME); public static final String GUIDE_POSTS = "GUIDE_POSTS"; public static final byte[] GUIDE_POSTS_BYTES = Bytes.toBytes(GUIDE_POSTS); + public static final String GUIDE_POSTS_COUNT = "GUIDE_POSTS_COUNT"; + public static final byte[] GUIDE_POSTS_COUNT_BYTES = Bytes.toBytes(GUIDE_POSTS_COUNT); + public static final String GUIDE_POSTS_WIDTH = "GUIDE_POSTS_WIDTH"; + public static final byte[] GUIDE_POSTS_WIDTH_BYTES = Bytes.toBytes(GUIDE_POSTS_WIDTH); public static final String MIN_KEY = "MIN_KEY"; public static final byte[] MIN_KEY_BYTES = Bytes.toBytes(MIN_KEY); public static final String MAX_KEY = "MAX_KEY"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index f4cf135..b26b426 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -35,6 +35,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_COUNT; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; @@ -235,7 +237,9 @@ public interface QueryConstants { PHYSICAL_NAME + " VARCHAR NOT NULL," + COLUMN_FAMILY + " VARCHAR," + REGION_NAME + " VARCHAR," + - GUIDE_POSTS + " VARBINARY[]," + + GUIDE_POSTS_COUNT + " BIGINT," + + GUIDE_POSTS + " VARBINARY," + + GUIDE_POSTS_WIDTH + " BIGINT," + MIN_KEY + " VARBINARY," + MAX_KEY + " VARBINARY," + LAST_STATS_UPDATE_TIME+ " DATE, "+ http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 007abd2..6733511 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -45,6 +45,7 @@ import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; +import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.schema.stats.PTableStatsImpl; import org.apache.phoenix.util.ByteUtil; @@ -375,7 +376,8 @@ public class PTableImpl implements PTable { } this.parentTableName = parentTableName; - this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName(schemaName.getString(), parentTableName.getString())); + this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName( + schemaName.getString(), parentTableName.getString())); estimatedSize += PNameFactory.getEstimatedSize(this.parentName); this.physicalNames = physicalNames == null ? ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames); @@ -705,7 +707,7 @@ public class PTableImpl implements PTable { public long getTimeStamp() { return timeStamp; } - + @Override public void readFields(DataInput input) throws IOException { byte[] tenantIdBytes = Bytes.readByteArray(input); http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java index d902cb9..9719ecf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java @@ -40,7 +40,7 @@ public interface PTableStats extends Writable { public static final PTableStats EMPTY_STATS = new PTableStats() { @Override - public SortedMap<byte[], List<byte[]>> getGuidePosts() { + public SortedMap<byte[], GuidePostsInfo> getGuidePosts() { return ImmutableSortedMap.of(); } @@ -65,7 +65,7 @@ public interface PTableStats extends Writable { * Returns a tree map of the guide posts collected against a column family * @return */ - SortedMap<byte[], List<byte[]>> getGuidePosts(); + SortedMap<byte[], GuidePostsInfo> getGuidePosts(); int getEstimatedSize(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index 159fb79..fab9f52 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -38,20 +38,20 @@ import com.sun.istack.NotNull; * Implementation for PTableStats */ public class PTableStatsImpl implements PTableStats { - private final SortedMap<byte[], List<byte[]>> guidePosts; + private final SortedMap<byte[], GuidePostsInfo> guidePosts; private final int estimatedSize; public PTableStatsImpl() { - this(new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR)); + this(new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR)); } - public PTableStatsImpl(@NotNull SortedMap<byte[], List<byte[]>> guidePosts) { + public PTableStatsImpl(@NotNull SortedMap<byte[], GuidePostsInfo> guidePosts) { this.guidePosts = guidePosts; int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size()); - for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) { + for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) { byte[] cf = entry.getKey(); estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; - List<byte[]> keys = entry.getValue(); + List<byte[]> keys = entry.getValue().getGuidePosts(); estimatedSize += SizedUtil.sizeOfArrayList(keys.size()); for (byte[] key : keys) { estimatedSize += SizedUtil.ARRAY_SIZE + key.length; @@ -61,16 +61,17 @@ public class PTableStatsImpl implements PTableStats { } @Override - public SortedMap<byte[], List<byte[]>> getGuidePosts() { + public SortedMap<byte[], GuidePostsInfo> getGuidePosts() { return guidePosts; } @Override public void write(DataOutput output) throws IOException { WritableUtils.writeVInt(output, guidePosts.size()); - for (Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) { + for (Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) { Bytes.writeByteArray(output, entry.getKey()); - List<byte[]> value = entry.getValue(); + List<byte[]> value = entry.getValue().getGuidePosts(); + output.writeLong(entry.getValue().getByteCount()); WritableUtils.writeVInt(output, value.size()); for (int i = 0; i < value.size(); i++) { Bytes.writeByteArray(output, value.get(i)); @@ -83,12 +84,13 @@ public class PTableStatsImpl implements PTableStats { int size = WritableUtils.readVInt(input); for (int i = 0; i < size; i++) { byte[] key = Bytes.readByteArray(input); + long byteCount = input.readLong(); int valueSize = WritableUtils.readVInt(input); List<byte[]> value = Lists.newArrayListWithExpectedSize(valueSize); for (int j = 0; j < valueSize; j++) { value.add(j, Bytes.readByteArray(input)); } - guidePosts.put(key, value); + guidePosts.put(key, new GuidePostsInfo(byteCount, value)); } } @@ -96,10 +98,10 @@ public class PTableStatsImpl implements PTableStats { public String toString() { StringBuilder buf = new StringBuilder(); buf.append("PTableStats ["); - for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) { + for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) { buf.append(Bytes.toStringBinary(entry.getKey())); buf.append(":("); - List<byte[]> keys = entry.getValue(); + List<byte[]> keys = entry.getValue().getGuidePosts(); if (!keys.isEmpty()) { for (byte[] key : keys) { buf.append(Bytes.toStringBinary(key)); @@ -117,5 +119,4 @@ public class PTableStatsImpl implements PTableStats { public int getEstimatedSize() { return estimatedSize; } - } http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 29706c7..2e7bfd9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -10,17 +10,20 @@ package org.apache.phoenix.schema.stats; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.KeyValueScanner; @@ -34,8 +37,7 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.schema.PDataType; -import org.apache.phoenix.schema.PhoenixArray; +import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.TimeKeeper; import com.google.common.collect.Lists; @@ -53,7 +55,8 @@ public class StatisticsCollector { private Map<String, byte[]> minMap = Maps.newHashMap(); private Map<String, byte[]> maxMap = Maps.newHashMap(); private long guidepostDepth; - private Map<String, Pair<Integer,List<byte[]>>> guidePostsMap = Maps.newHashMap(); + private Map<String, Pair<Long,GuidePostsInfo>> guidePostsMap = Maps.newHashMap(); + // Tracks the bytecount per family if it has reached the guidePostsDepth private Map<ImmutableBytesPtr, Boolean> familyMap = Maps.newHashMap(); protected StatisticsWriter statsTable; // Ensures that either analyze or compaction happens at any point of time. @@ -63,11 +66,15 @@ public class StatisticsCollector { throws IOException { Configuration config = env.getConfiguration(); HTableInterface statsHTable = env.getTable((PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES)); - guidepostDepth = - config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, - statsHTable.getTableDescriptor().getMaxFileSize() / - config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, - QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION)); + long maxFileSize = statsHTable.getTableDescriptor().getMaxFileSize(); + if (maxFileSize <= 0) { // HBase brain dead API doesn't give you the "real" max file size if it's not set... + maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE; + } + guidepostDepth = config.getLong( + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, + maxFileSize + / config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, + QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION)); // Get the stats table associated with the current table on which the CP is // triggered @@ -262,7 +269,8 @@ public class StatisticsCollector { familyMap.put(new ImmutableBytesPtr(cf), true); String fam = Bytes.toString(cf); - byte[] row = new ImmutableBytesPtr(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()).copyBytesIfNecessary(); + byte[] row = ByteUtil.copyKeyBytesIfNecessary( + new ImmutableBytesWritable(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())); if (!minMap.containsKey(fam) && !maxMap.containsKey(fam)) { minMap.put(fam, row); // Ideally the max key also should be added in this case @@ -278,19 +286,17 @@ public class StatisticsCollector { } } // TODO : This can be moved to an interface so that we could collect guide posts in different ways - Pair<Integer,List<byte[]>> gps = guidePostsMap.get(fam); + Pair<Long,GuidePostsInfo> gps = guidePostsMap.get(fam); if (gps == null) { - gps = new Pair<Integer,List<byte[]>>(0, Lists.<byte[]>newArrayList()); + gps = new Pair<Long,GuidePostsInfo>(0L,new GuidePostsInfo(0, Collections.<byte[]>emptyList())); guidePostsMap.put(fam, gps); } - int byteCount = gps.getFirst() + kv.getLength(); + int kvLength = kv.getLength(); + long byteCount = gps.getFirst() + kvLength; gps.setFirst(byteCount); if (byteCount >= guidepostDepth) { - // Prevent dups - List<byte[]> gpsKeys = gps.getSecond(); - if (gpsKeys.isEmpty() || Bytes.compareTo(row, gpsKeys.get(gpsKeys.size()-1)) > 0) { - gpsKeys.add(row); - gps.setFirst(0); // Only reset count when adding guidepost + if (gps.getSecond().addGuidePost(row, byteCount)) { + gps.setFirst(0L); } } } @@ -305,22 +311,10 @@ public class StatisticsCollector { return null; } - public byte[] getGuidePosts(String fam) { - if (!guidePostsMap.isEmpty()) { - Pair<Integer,List<byte[]>> gps = guidePostsMap.get(fam); - if (gps != null) { - List<byte[]> guidePosts = gps.getSecond(); - if (!guidePosts.isEmpty()) { - byte[][] array = new byte[guidePosts.size()][]; - int i = 0; - for (byte[] element : guidePosts) { - array[i] = element; - i++; - } - PhoenixArray phoenixArray = new PhoenixArray(PDataType.VARBINARY, array); - return PDataType.VARBINARY_ARRAY.toBytes(phoenixArray); - } - } + public GuidePostsInfo getGuidePosts(String fam) { + Pair<Long,GuidePostsInfo> pair = guidePostsMap.get(fam); + if (pair != null) { + return pair.getSecond(); } return null; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index bd59730..8d7dd00 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -19,8 +19,6 @@ package org.apache.phoenix.schema.stats; import static org.apache.phoenix.util.SchemaUtil.getVarCharLength; import java.io.IOException; -import java.util.Collections; -import java.util.List; import java.util.TreeMap; import org.apache.hadoop.hbase.KeyValue; @@ -33,12 +31,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.schema.PDataType; -import org.apache.phoenix.schema.PhoenixArray; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.MetaDataUtil; - -import com.google.common.collect.Lists; /** * Simple utility class for managing multiple key parts of the statistic */ @@ -71,7 +65,7 @@ public class StatisticsUtil { ResultScanner scanner = statsHTable.getScanner(s); try { Result result = null; - TreeMap<byte[], List<byte[]>> guidePostsPerCf = new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR); + TreeMap<byte[], GuidePostsInfo> guidePostsPerCf = new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR); while ((result = scanner.next()) != null) { KeyValue current = result.raw()[0]; int tableNameLength = tableNameBytes.length + 1; @@ -79,28 +73,13 @@ public class StatisticsUtil { int cfLength = getVarCharLength(current.getBuffer(), cfOffset, current.getRowLength() - tableNameLength); ptr.set(current.getBuffer(), cfOffset, cfLength); byte[] cfName = ByteUtil.copyKeyBytesIfNecessary(ptr); - PhoenixArray array = (PhoenixArray)PDataType.VARBINARY_ARRAY.toObject(current.getBuffer(), current.getValueOffset(), current - .getValueLength()); - if (array != null && array.getDimensions() != 0) { - List<byte[]> guidePosts = Lists.newArrayListWithExpectedSize(array.getDimensions()); - for (int j = 0; j < array.getDimensions(); j++) { - byte[] gp = array.toBytes(j); - if (gp.length != 0) { - guidePosts.add(gp); - } - } - List<byte[]> gps = guidePostsPerCf.put(cfName, guidePosts); - if (gps != null) { // Add guidepost already there from other regions - guidePosts.addAll(gps); - } + GuidePostsInfo newInfo = GuidePostsInfo.fromBytes(current.getBuffer(), current.getValueOffset(), current.getValueLength()); + GuidePostsInfo oldInfo = guidePostsPerCf.put(cfName, newInfo); + if (oldInfo != null) { + newInfo.combine(oldInfo); } } if (!guidePostsPerCf.isEmpty()) { - // Sort guideposts, as the order above will depend on the order we traverse - // each region's worth of guideposts above. - for (List<byte[]> gps : guidePostsPerCf.values()) { - Collections.sort(gps, Bytes.BYTES_COMPARATOR); - } return new PTableStatsImpl(guidePostsPerCf); } } finally { http://git-wip-us.apache.org/repos/asf/phoenix/blob/75484fb3/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 5bf39c5..3f391f5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -87,9 +87,14 @@ public class StatisticsWriter implements Closeable { byte[] prefix = StatisticsUtil.getRowKey(tableName, PDataType.VARCHAR.toBytes(fam), PDataType.VARCHAR.toBytes(regionName)); Put put = new Put(prefix); - if (tracker.getGuidePosts(fam) != null) { + GuidePostsInfo gp = tracker.getGuidePosts(fam); + if (gp != null) { + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_COUNT_BYTES, + clientTimeStamp, PDataType.LONG.toBytes((gp.getGuidePosts().size()))); put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_BYTES, - clientTimeStamp, (tracker.getGuidePosts(fam))); + clientTimeStamp, PDataType.VARBINARY.toBytes(gp.toBytes())); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, + clientTimeStamp, PDataType.LONG.toBytes(gp.getByteCount())); } put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.MIN_KEY_BYTES, clientTimeStamp, PDataType.VARBINARY.toBytes(tracker.getMinKey(fam)));
