PHOENIX-2145 Pherf - Make update stats optional and fix threads not exiting after performance run
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4c945d3c Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4c945d3c Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4c945d3c Branch: refs/heads/4.5-HBase-1.0 Commit: 4c945d3c551ecb27fd0ad3af5d6737720f5e6c51 Parents: 01825cc Author: Mujtaba <mujt...@apache.org> Authored: Tue Aug 4 15:47:44 2015 -0700 Committer: James Taylor <jtay...@salesforce.com> Committed: Tue Aug 11 18:42:11 2015 -0700 ---------------------------------------------------------------------- phoenix-pherf/pom.xml | 3 -- .../org/apache/phoenix/pherf/DataIngestIT.java | 4 +- .../src/main/assembly/components-minimal.xml | 2 +- .../java/org/apache/phoenix/pherf/Pherf.java | 23 +++++------ .../apache/phoenix/pherf/PherfConstants.java | 5 +++ .../apache/phoenix/pherf/util/PhoenixUtil.java | 11 +++++ .../phoenix/pherf/workload/QueryExecutor.java | 3 +- .../phoenix/pherf/workload/WriteWorkload.java | 43 ++++++++++---------- .../org/apache/phoenix/pherf/PherfTest.java | 2 +- 9 files changed, 55 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml index 281c771..bd92c9e 100644 --- a/phoenix-pherf/pom.xml +++ b/phoenix-pherf/pom.xml @@ -108,9 +108,6 @@ </resource> <resource> <directory>config</directory> - <includes> - <include>**/*.properties</include> - </includes> </resource> </resources> <testResources> http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java index 8a340b3..1defbb1 100644 --- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java @@ -20,6 +20,8 @@ package org.apache.phoenix.pherf; import com.jcabi.jdbc.JdbcSession; import com.jcabi.jdbc.Outcome; + +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.DataTypeMapping; @@ -66,7 +68,7 @@ public class DataIngestIT extends ResultBaseTestIT { scenario.getTableNameWithoutSchemaName(), util.getConnection()); assertTrue("Could not get phoenix columns.", columnListFromPhoenix.size() > 0); - WriteWorkload loader = new WriteWorkload(util, parser, scenario); + WriteWorkload loader = new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO); WorkloadExecutor executor = new WorkloadExecutor(); executor.add(loader); executor.get(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/assembly/components-minimal.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/assembly/components-minimal.xml b/phoenix-pherf/src/main/assembly/components-minimal.xml index 38c6929..9c9b9a1 100644 --- a/phoenix-pherf/src/main/assembly/components-minimal.xml +++ b/phoenix-pherf/src/main/assembly/components-minimal.xml @@ -23,7 +23,7 @@ <outputDirectory>/</outputDirectory> <includes> <include>org.apache.phoenix:phoenix-pherf</include> - <include>org.jfree:jfreechart</include> + <include>org.jfree:*</include> <include>com.googlecode.java-diff-utils:diffutils</include> <include>org.apache.commons:commons-lang3</include> <include>org.apache.commons:commons-math3</include> http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java index 5a9f45f..0421b6f 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java @@ -19,6 +19,7 @@ package org.apache.phoenix.pherf; import org.apache.commons.cli.*; +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.XMLConfigParser; import org.apache.phoenix.pherf.jmx.MonitorManager; import org.apache.phoenix.pherf.schema.SchemaReader; @@ -73,6 +74,8 @@ public class Pherf { + "See pherf.default.dataloader.threadpool in Pherf.properties."); options.addOption("h", "help", false, "Get help on using this utility."); options.addOption("d", "debug", false, "Put tool in debug mode"); + options.addOption("stats", false, + "Update Phoenix Statistics after data is loaded with -l argument"); } private final String zookeeper; @@ -89,6 +92,7 @@ public class Pherf { private final int rowCountOverride; private final boolean listFiles; private final boolean applySchema; + private final GeneratePhoenixStats generateStatistics; public Pherf(String[] args) throws Exception { CommandLineParser parser = new PosixParser(); @@ -126,6 +130,7 @@ public class Pherf { command.hasOption("scenarioFile") ? command.getOptionValue("scenarioFile") : null; schemaFile = command.hasOption("schemaFile") ? command.getOptionValue("schemaFile") : null; rowCountOverride = Integer.parseInt(command.getOptionValue("rowCountOverride", "0")); + generateStatistics = command.hasOption("stats") ? GeneratePhoenixStats.YES : GeneratePhoenixStats.NO; String writerThreadPoolSize = command.getOptionValue("writerThreadSize", @@ -204,23 +209,11 @@ public class Pherf { // Schema and Data Load if (preLoadData) { logger.info("\nStarting Data Load..."); - WriteWorkload workload = new WriteWorkload(parser); + WriteWorkload workload = new WriteWorkload(parser, generateStatistics); workloadExecutor.add(workload); // Wait for dataLoad to complete workloadExecutor.get(workload); - - logger.info("\nGenerate query gold files after data load"); - QueryExecutor - goldFileGenerator = - new QueryExecutor(parser, phoenixUtil, workloadExecutor.getPool(), - parser.getDataModels(), queryHint, true, - PherfConstants.RunMode.FUNCTIONAL); - workloadExecutor - .add(goldFileGenerator); - - // Wait for dataLoad to complete - workloadExecutor.get(goldFileGenerator); } else { logger.info( "\nSKIPPED: Data Load and schema creation as -l argument not specified"); @@ -254,6 +247,10 @@ public class Pherf { if (workloadExecutor != null) { logger.info("Run completed. Shutting down thread pool."); workloadExecutor.shutdown(); + if (preLoadData) { + System.exit(0); + } + } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java index e060e53..12580d4 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java @@ -62,6 +62,11 @@ public class PherfConstants { public static final int MONITOR_FREQUENCY = 5000; public static final String MONITOR_FILE_NAME = "STATS_MONITOR"; + public static enum GeneratePhoenixStats { + YES, + NO + } + public static enum RunMode { PERFORMANCE, FUNCTIONAL http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java index 5f7d637..5b223b1 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java @@ -255,4 +255,15 @@ public class PhoenixUtil { public static void setRowCountOverride(int rowCountOverride) { PhoenixUtil.rowCountOverride = rowCountOverride; } + + /** + * Update Phoenix table stats + * + * @param tableName + * @throws Exception + */ + public void updatePhoenixStats(String tableName) throws Exception { + logger.info("Updating stats for " + tableName); + executeStatement("UPDATE STATISTICS " + tableName); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java index 624188c..4ab76c8 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java @@ -20,6 +20,7 @@ package org.apache.phoenix.pherf.workload; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.PherfConstants.RunMode; import org.apache.phoenix.pherf.configuration.*; import org.apache.phoenix.pherf.result.*; @@ -142,7 +143,7 @@ public class QueryExecutor implements Workload { for (int i = 0; i < writerThreadCount; i++) { logger.debug("Inserting write workload ( " + i + " ) of ( " + writerThreadCount + " )"); - Workload writes = new WriteWorkload(PhoenixUtil.create(), parser); + Workload writes = new WriteWorkload(PhoenixUtil.create(), parser, GeneratePhoenixStats.NO); pool.submit(writes.execute()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java index 305521b..b6686c6 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java @@ -19,6 +19,7 @@ package org.apache.phoenix.pherf.workload; import org.apache.phoenix.pherf.PherfConstants; +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.configuration.WriteParams; @@ -58,19 +59,24 @@ public class WriteWorkload implements Workload { private final int threadPoolSize; private final int batchSize; + private final GeneratePhoenixStats generateStatistics; public WriteWorkload(XMLConfigParser parser) throws Exception { - this(PhoenixUtil.create(), parser); + this(PhoenixUtil.create(), parser, GeneratePhoenixStats.NO); + } + + public WriteWorkload(XMLConfigParser parser, GeneratePhoenixStats generateStatistics) throws Exception { + this(PhoenixUtil.create(), parser, generateStatistics); } - public WriteWorkload(PhoenixUtil util, XMLConfigParser parser) throws Exception { - this(util, parser, null); + public WriteWorkload(PhoenixUtil util, XMLConfigParser parser, GeneratePhoenixStats generateStatistics) throws Exception { + this(util, parser, null, generateStatistics); } - public WriteWorkload(PhoenixUtil phoenixUtil, XMLConfigParser parser, Scenario scenario) + public WriteWorkload(PhoenixUtil phoenixUtil, XMLConfigParser parser, Scenario scenario, GeneratePhoenixStats generateStatistics) throws Exception { this(phoenixUtil, PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES), - parser, scenario); + parser, scenario, generateStatistics); } /** @@ -87,11 +93,12 @@ public class WriteWorkload implements Workload { * @throws Exception */ public WriteWorkload(PhoenixUtil phoenixUtil, Properties properties, XMLConfigParser parser, - Scenario scenario) throws Exception { + Scenario scenario, GeneratePhoenixStats generateStatistics) throws Exception { this.pUtil = phoenixUtil; this.parser = parser; this.rulesApplier = new RulesApplier(parser); this.resultUtil = new ResultUtil(); + this.generateStatistics = generateStatistics; // Overwrite defaults properties with those given in the configuration. This indicates the // scenario is a R/W mixed workload. @@ -156,9 +163,15 @@ public class WriteWorkload implements Workload { List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario); waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches); - - // always update stats for Phoenix base tables - updatePhoenixStats(scenario.getTableName()); + + // Update Phoenix Statistics + if (this.generateStatistics == GeneratePhoenixStats.YES) { + logger.info("Updating Phoenix table statistics..."); + pUtil.updatePhoenixStats(scenario.getTableName()); + logger.info("Stats update done!"); + } else { + logger.info("Phoenix table stats update not requested."); + } } private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario) @@ -208,18 +221,6 @@ public class WriteWorkload implements Workload { .add(scenario.getTableName(), sumRows, (int) (System.currentTimeMillis() - start)); } - /** - * TODO Move this method to PhoenixUtil - * Update Phoenix table stats - * - * @param tableName - * @throws Exception - */ - public void updatePhoenixStats(String tableName) throws Exception { - logger.info("Updating stats for " + tableName); - pUtil.executeStatement("UPDATE STATISTICS " + tableName); - } - public Future<Info> upsertData(final Scenario scenario, final List<Column> columns, final String tableName, final int rowCount, final DataLoadThreadTime dataLoadThreadTime) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c945d3c/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java index 94c8171..731a57a 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java @@ -34,7 +34,7 @@ public class PherfTest { @Test public void testUnknownOption() { - String[] args = {"-drop", "all", "-l", "-q", "-m","-bsOption"}; + String[] args = {"-drop", "all", "-q", "-m","-bsOption"}; // Makes sure that System.exit(1) is called. Release is a required param. exit.expectSystemExitWithStatus(1);
