PHOENIX-1746 Pass through guidepost config params on UPDATE STATISTICS call
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/096586e6 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/096586e6 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/096586e6 Branch: refs/heads/4.x-HBase-1.x Commit: 096586e65e2779433bf30c30e97f78ae2316365e Parents: e06ceaf Author: James Taylor <[email protected]> Authored: Mon Mar 23 10:28:23 2015 -0700 Committer: James Taylor <[email protected]> Committed: Mon Mar 23 10:29:29 2015 -0700 ---------------------------------------------------------------------- .../StatsCollectorWithSplitsAndMultiCFIT.java | 6 ++++++ phoenix-core/src/main/antlr3/PhoenixSQL.g | 4 ++-- .../coprocessor/BaseScannerRegionObserver.java | 6 ++++-- .../UngroupedAggregateRegionObserver.java | 4 +++- .../apache/phoenix/jdbc/PhoenixStatement.java | 9 +++++---- .../apache/phoenix/parse/ParseNodeFactory.java | 4 ++-- .../parse/UpdateStatisticsStatement.java | 11 +++++++++- .../apache/phoenix/schema/MetaDataClient.java | 19 ++++++++++++++---- .../schema/stats/StatisticsCollector.java | 21 +++++++++++++++----- 9 files changed, 63 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java index c34d598..bcb3a0a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java @@ -136,6 +136,12 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract assertRowCountAndByteCount(info, rowCountArr[i], byteCountArr[i]); i++; } + + TestUtil.analyzeTable(conn, STATS_TEST_TABLE_NAME_NEW); + String query = "UPDATE STATISTICS " + STATS_TEST_TABLE_NAME_NEW + " SET \"" + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(2000); + conn.createStatement().execute(query); + keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME_NEW); + assertEquals(6, keyRanges.size()); } protected void assertRowCountAndByteCount(GuidePostsInfo info, long rowCount, long byteCount) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/antlr3/PhoenixSQL.g ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index 6a2e3b9..0330a39 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -520,8 +520,8 @@ alter_table_node returns [AlterTableStatement ret] ; update_statistics_node returns [UpdateStatisticsStatement ret] - : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)? - {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())));} + : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)? (SET (p=properties))? + {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())), p);} ; prop_name returns [String ret] http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index c3988a0..a2269b4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -38,6 +38,8 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.htrace.Span; +import org.apache.htrace.Trace; import org.apache.phoenix.execute.TupleProjector; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; @@ -53,8 +55,6 @@ import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.ServerUtil; -import org.apache.htrace.Span; -import org.apache.htrace.Trace; import com.google.common.collect.ImmutableList; @@ -85,6 +85,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String EXPECTED_UPPER_REGION_KEY = "_ExpectedUpperRegionKey"; public static final String REVERSE_SCAN = "_ReverseScan"; public static final String ANALYZE_TABLE = "_ANALYZETABLE"; + public static final String GUIDEPOST_WIDTH_BYTES = "_GUIDEPOST_WIDTH_BYTES"; + public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION"; /** * Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations * are used to augment log lines emitted by Phoenix. See https://issues.apache.org/jira/browse/PHOENIX-1198. http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index fc37a84..e43e5e5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -162,8 +162,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{ long ts = scan.getTimeRange().getMax(); StatisticsCollector stats = null; if(ScanUtil.isAnalyzeTable(scan)) { + byte[] gp_width_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES); + byte[] gp_per_region_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION); // Let this throw, as this scan is being done for the sole purpose of collecting stats - stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts); + stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts, gp_width_bytes, gp_per_region_bytes); } if (ScanUtil.isLocalIndex(scan)) { /* http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 996d243..f802ff4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Pair; @@ -717,8 +718,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho private static class ExecutableUpdateStatisticsStatement extends UpdateStatisticsStatement implements CompilableStatement { - public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope) { - super(table, scope); + public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) { + super(table, scope, props); } @SuppressWarnings("unchecked") @@ -919,8 +920,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho } @Override - public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) { - return new ExecutableUpdateStatisticsStatement(table, scope); + public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) { + return new ExecutableUpdateStatisticsStatement(table, scope, props); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index 82ae821..931f327 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -357,8 +357,8 @@ public class ParseNodeFactory { return new DivideParseNode(children); } - public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) { - return new UpdateStatisticsStatement(table, scope); + public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) { + return new UpdateStatisticsStatement(table, scope, props); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java index dff9f06..6f7b736 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java @@ -21,6 +21,8 @@ import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.ALL; import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.COLUMNS; import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.INDEX; +import java.util.Map; + import org.apache.phoenix.schema.stats.StatisticsCollectionScope; import com.sun.istack.NotNull; @@ -28,9 +30,12 @@ import com.sun.istack.NotNull; public class UpdateStatisticsStatement extends SingleTableStatement { private final StatisticsCollectionScope scope; - public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope) { + private final Map<String,Object> props; + + public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope, Map<String,Object> props) { super(table, 0); this.scope = scope; + this.props = props; } public boolean updateColumns() { @@ -43,5 +48,9 @@ public class UpdateStatisticsStatement extends SingleTableStatement { public boolean updateAll() { return scope == ALL; + } + + public Map<String,Object> getProps() { + return props; }; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 7688531..2ba0cde 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -153,6 +153,7 @@ import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTable.ViewType; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; @@ -618,7 +619,7 @@ public class MetaDataClient { PTable table = resolver.getTables().get(0).getTable(); long rowCount = 0; if (updateStatisticsStmt.updateColumns()) { - rowCount += updateStatisticsInternal(table.getPhysicalName(), table); + rowCount += updateStatisticsInternal(table.getPhysicalName(), table, updateStatisticsStmt.getProps()); } if (updateStatisticsStmt.updateIndex()) { // TODO: If our table is a VIEW with multiple indexes or a TABLE with local indexes, @@ -626,7 +627,7 @@ public class MetaDataClient { // across all indexes in that case so that we don't re-calculate the same stats // multiple times. for (PTable index : table.getIndexes()) { - rowCount += updateStatisticsInternal(index.getPhysicalName(), index); + rowCount += updateStatisticsInternal(index.getPhysicalName(), index, updateStatisticsStmt.getProps()); } // If analyzing the indexes of a multi-tenant table or a table with view indexes // then analyze all of those indexes too. @@ -654,14 +655,14 @@ public class MetaDataClient { return PTableStats.EMPTY_STATS; } }; - rowCount += updateStatisticsInternal(name, indexLogicalTable); + rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps()); } } } return new MutationState((int)rowCount, connection); } - private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException { + private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException { ReadOnlyProps props = connection.getQueryServices().getProps(); final long msMinBetweenUpdates = props .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, @@ -691,6 +692,16 @@ public class MetaDataClient { Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES); + if (statsProps != null) { + Object gp_width = statsProps.get(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB); + if (gp_width != null) { + scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES, PLong.INSTANCE.toBytes(gp_width)); + } + Object gp_per_region = statsProps.get(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB); + if (gp_per_region != null) { + scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION, PInteger.INSTANCE.toBytes(gp_per_region)); + } + } MutationState mutationState = plan.execute(); rowCount = mutationState.getUpdateCount(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index ade0fba..d6f25c4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -38,6 +38,9 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.TimeKeeper; import org.slf4j.Logger; @@ -65,15 +68,23 @@ public class StatisticsCollector { private Pair<Long,GuidePostsInfo> cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { - this(env, tableName, clientTimeStamp, null); + this(env, tableName, clientTimeStamp, null, null, null); + } + + public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException { + this(env, tableName, clientTimeStamp, null, gp_width_bytes, gp_per_region_bytes); } public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family) throws IOException { + this(env, tableName, clientTimeStamp, family, null, null); + } + + public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException { Configuration config = env.getConfiguration(); - int guidepostPerRegion = config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION); - long guidepostWidth = config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES); + int guidepostPerRegion = gp_per_region_bytes == null ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault()); + long guidepostWidth = gp_width_bytes == null ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault()); this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, env.getRegion().getTableDesc()); // Get the stats table associated with the current table on which the CP is // triggered
