DRILL-1407: Add scan size calculator option to HBase storage plugin configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9ac177c8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9ac177c8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9ac177c8 Branch: refs/heads/master Commit: 9ac177c817fcc7d79ae1069e73f7e8cad88dfc99 Parents: 525bc79 Author: Steven Phillips <sphill...@maprtech.com> Authored: Fri Sep 12 01:37:49 2014 -0700 Committer: Steven Phillips <sphill...@maprtech.com> Committed: Sun Sep 28 01:10:56 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/store/hbase/HBaseGroupScan.java | 2 +- .../exec/store/hbase/HBaseStoragePluginConfig.java | 15 ++++++++++++++- .../drill/exec/store/hbase/TableStatsCalculator.java | 13 +++++-------- 3 files changed, 20 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ac177c8/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java index 5a03063..1112de0 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java @@ -147,7 +147,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst HTable table = new HTable(storagePluginConfig.getHBaseConf(), hbaseScanSpec.getTableName()); this.hTableDesc = table.getTableDescriptor(); NavigableMap<HRegionInfo, ServerName> regionsMap = table.getRegionLocations(); - statsCalculator = new TableStatsCalculator(table, hbaseScanSpec, storagePlugin.getContext().getConfig()); + statsCalculator = new TableStatsCalculator(table, hbaseScanSpec, storagePlugin.getContext().getConfig(), storagePluginConfig); boolean foundStartRegion = false; regionsToScan = new TreeMap<HRegionInfo, ServerName>(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ac177c8/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java index d7d20a5..85f0d13 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java @@ -41,16 +41,24 @@ public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements @JsonIgnore private Configuration hbaseConf; + @JsonIgnore + private Boolean sizeCalculatorEnabled; + public static final String NAME = "hbase"; @JsonCreator - public HBaseStoragePluginConfig(@JsonProperty("config") Map<String, String> props) { + public HBaseStoragePluginConfig(@JsonProperty("config") Map<String, String> props, @JsonProperty("size.calculator.enabled") Boolean sizeCalculatorEnabled) { this.config = props; if (config == null) { config = Maps.newHashMap(); } logger.debug("Initializing HBase StoragePlugin configuration with zookeeper quorum '{}', port '{}'.", config.get(HConstants.ZOOKEEPER_QUORUM), config.get(HBASE_ZOOKEEPER_PORT)); + if (sizeCalculatorEnabled == null) { + this.sizeCalculatorEnabled = false; + } else { + this.sizeCalculatorEnabled = sizeCalculatorEnabled; + } } @JsonProperty @@ -58,6 +66,11 @@ public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements return ImmutableMap.copyOf(config); } + @JsonProperty("size.calculator.enabled") + public boolean isSizeCalculatorEnabled() { + return sizeCalculatorEnabled; + } + @Override public boolean equals(Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ac177c8/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/TableStatsCalculator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/TableStatsCalculator.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/TableStatsCalculator.java index 125483e..6a1f9f6 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/TableStatsCalculator.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/TableStatsCalculator.java @@ -46,8 +46,6 @@ public class TableStatsCalculator { private static final String DRILL_EXEC_HBASE_SCAN_SAMPLE_ROWS_COUNT = "drill.exec.hbase.scan.samplerows.count"; - private static final String DRILL_EXEC_HBASE_SCAN_SIZECALCULATOR_ENABLED = "drill.exec.hbase.scan.sizecalculator.enabled"; - private static final int DEFAULT_SAMPLE_SIZE = 100; /** @@ -64,10 +62,10 @@ public class TableStatsCalculator { * * @param table * @param hbaseScanSpec - * @param drillConfig + * @param config * @throws IOException */ - public TableStatsCalculator(HTable table, HBaseScanSpec hbaseScanSpec, DrillConfig config) throws IOException { + public TableStatsCalculator(HTable table, HBaseScanSpec hbaseScanSpec, DrillConfig config, HBaseStoragePluginConfig storageConfig) throws IOException { HBaseAdmin admin = new HBaseAdmin(table.getConfiguration()); try { int rowsToSample = rowsToSample(config); @@ -90,7 +88,7 @@ public class TableStatsCalculator { scanner.close(); } - if (!enabled(config)) { + if (!enabled(storageConfig)) { logger.info("Region size calculation disabled."); return; } @@ -141,9 +139,8 @@ public class TableStatsCalculator { } - private boolean enabled(DrillConfig config) { - return config.hasPath(DRILL_EXEC_HBASE_SCAN_SIZECALCULATOR_ENABLED) - ? config.getBoolean(DRILL_EXEC_HBASE_SCAN_SIZECALCULATOR_ENABLED) : true; + private boolean enabled(HBaseStoragePluginConfig config) { + return config.isSizeCalculatorEnabled(); } private int rowsToSample(DrillConfig config) {
