Repository: phoenix Updated Branches: refs/heads/master d93ad61cb -> 073392e35
PHOENIX-4549 Pherf - Column override and sequenced index creation support Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/073392e3 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/073392e3 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/073392e3 Branch: refs/heads/master Commit: 073392e35d740e06748500b554ee60bf755de349 Parents: d93ad61 Author: Mujtaba <mujt...@apache.org> Authored: Wed Feb 7 11:49:15 2018 -0800 Committer: Mujtaba <mujt...@apache.org> Committed: Wed Feb 7 11:49:15 2018 -0800 ---------------------------------------------------------------------- .../phoenix/pherf/configuration/Column.java | 23 +-- .../pherf/configuration/DataTypeMapping.java | 6 +- .../phoenix/pherf/configuration/Query.java | 25 ++++ .../phoenix/pherf/configuration/Scenario.java | 3 +- .../phoenix/pherf/result/QueryResult.java | 10 +- .../phoenix/pherf/result/ResultManager.java | 18 ++- .../apache/phoenix/pherf/result/ResultUtil.java | 9 +- .../phoenix/pherf/rules/RulesApplier.java | 142 +++++++++++++++---- .../phoenix/pherf/schema/SchemaReader.java | 2 +- .../apache/phoenix/pherf/util/PhoenixUtil.java | 44 +++++- .../pherf/workload/MultiThreadedRunner.java | 35 ++++- .../phoenix/pherf/workload/QueryExecutor.java | 36 ++--- .../phoenix/pherf/workload/WriteWorkload.java | 54 ++++++- .../scenario/prod_test_unsalted_scenario.xml | 14 +- .../org/apache/phoenix/pherf/ColumnTest.java | 3 + .../phoenix/pherf/ConfigurationParserTest.java | 4 +- .../apache/phoenix/pherf/RuleGeneratorTest.java | 107 +++++++++++++- .../test/resources/datamodel/test_schema.sql | 4 + .../test/resources/scenario/test_scenario.xml | 48 ++++++- 19 files changed, 484 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java index 7c9e180..0d64a39 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Column.java @@ -28,7 +28,8 @@ public class Column { private String name; private String prefix; private DataSequence dataSequence; - private int length, minValue, maxValue, precision; + private int length, precision; + private long minValue, maxValue; private int nullChance; private boolean userDefined; private List<DataValue> dataValues; @@ -40,8 +41,8 @@ public class Column { // Initialize int to negative value so we can distinguish 0 in mutations // Object fields can be detected with null this.length = Integer.MIN_VALUE; - this.minValue = Integer.MIN_VALUE; - this.maxValue = Integer.MIN_VALUE; + this.minValue = Long.MIN_VALUE; + this.maxValue = Long.MIN_VALUE; this.precision = Integer.MIN_VALUE; this.nullChance = Integer.MIN_VALUE; this.userDefined = false; @@ -84,6 +85,10 @@ public class Column { public int getLength() { return length; } + + public int getLengthExcludingPrefix() { + return (this.getPrefix() == null) ? this.length : this.length - this.getPrefix().length(); + } public void setLength(int length) { this.length = length; @@ -97,19 +102,19 @@ public class Column { this.type = type; } - public int getMinValue() { + public long getMinValue() { return minValue; } - public void setMinValue(int minValue) { + public void setMinValue(long minValue) { this.minValue = minValue; } - public int getMaxValue() { + public long getMaxValue() { return maxValue; } - public void setMaxValue(int maxValue) { + public void setMaxValue(long maxValue) { this.maxValue = maxValue; } @@ -134,11 +139,11 @@ public class Column { * obj contains only the fields you want to mutate this object into. */ public void mutate(Column column) { - if (column.getMinValue() != Integer.MIN_VALUE) { + if (column.getMinValue() != Long.MIN_VALUE) { setMinValue(column.getMinValue()); } - if (column.getMaxValue() != Integer.MIN_VALUE) { + if (column.getMaxValue() != Long.MIN_VALUE) { setMaxValue(column.getMaxValue()); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java index c266a57..0476df2 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java @@ -25,7 +25,11 @@ public enum DataTypeMapping { CHAR("CHAR", Types.CHAR), DECIMAL("DECIMAL", Types.DECIMAL), INTEGER("INTEGER", Types.INTEGER), - DATE("DATE", Types.DATE); + DATE("DATE", Types.DATE), + UNSIGNED_LONG("UNSIGNED_LONG", Types.LONGVARCHAR), + VARCHAR_ARRAY("VARCHAR ARRAY", Types.ARRAY), + VARBINARY("VARBINARY", Types.VARBINARY), + TIMESTAMP("TIMESTAMP", Types.TIMESTAMP); private final String sType; http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java index 1e5cabe..e283715 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java @@ -18,9 +18,14 @@ package org.apache.phoenix.pherf.configuration; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlType; +import org.apache.phoenix.pherf.rules.RulesApplier; + @XmlType public class Query { @@ -30,7 +35,12 @@ public class Query { private String ddl; private String queryGroup; private String id; + private Pattern pattern; + public Query() { + pattern = Pattern.compile("\\[.*?\\]"); + } + /** * SQL statement * @@ -40,6 +50,21 @@ public class Query { public String getStatement() { return statement; } + + public String getDynamicStatement(RulesApplier ruleApplier, Scenario scenario) throws Exception { + String ret = this.statement; + String needQuotes = ""; + Matcher m = pattern.matcher(ret); + while(m.find()) { + String dynamicField = m.group(0).replace("[", "").replace("]", ""); + Column dynamicColumn = ruleApplier.getRule(dynamicField, scenario); + needQuotes = (dynamicColumn.getType() == DataTypeMapping.CHAR || dynamicColumn + .getType() == DataTypeMapping.VARCHAR) ? "'" : ""; + ret = ret.replace("[" + dynamicField + "]", + needQuotes + ruleApplier.getDataValue(dynamicColumn).getValue() + needQuotes); + } + return ret; + } public void setStatement(String statement) { // normalize statement - merge all consecutive spaces into one http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java index 02e5cc7..132207b 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java @@ -37,7 +37,7 @@ public class Scenario { private Map<String, String> phoenixProperties; private DataOverride dataOverride; private List<QuerySet> querySet = new ArrayList<>(); - private WriteParams writeParams; + private WriteParams writeParams = null; private String name; private String tenantId; private List<Ddl> preScenarioDdls; @@ -45,7 +45,6 @@ public class Scenario { public Scenario() { - writeParams = new WriteParams(); } /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/QueryResult.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/QueryResult.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/QueryResult.java index 669a472..cef24f4 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/QueryResult.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/QueryResult.java @@ -19,7 +19,9 @@ package org.apache.phoenix.pherf.result; import org.apache.phoenix.pherf.configuration.Query; +import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.result.file.ResultFileDetails; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.util.DateUtil; @@ -99,7 +101,7 @@ public class QueryResult extends Query { return totalRunTime / getThreadTimes().size(); } - public List<ResultValue> getCsvRepresentation(ResultUtil util) { + public List<ResultValue> getCsvRepresentation(ResultUtil util, Scenario scenario, RulesApplier ruleApplier) { List<ResultValue> rowValues = new ArrayList<>(); rowValues.add(new ResultValue(util.convertNull(getStartTimeText()))); rowValues.add(new ResultValue(util.convertNull(this.getQueryGroup()))); @@ -109,14 +111,14 @@ public class QueryResult extends Query { rowValues.add(new ResultValue(util.convertNull(String.valueOf(getAvgRunTimeInMs())))); rowValues.add(new ResultValue(util.convertNull(String.valueOf(getAvgMinRunTimeInMs())))); rowValues.add(new ResultValue(util.convertNull(String.valueOf(getRunCount())))); - rowValues.add(new ResultValue(util.convertNull(String.valueOf(getExplainPlan())))); + rowValues.add(new ResultValue(util.convertNull(String.valueOf(getExplainPlan(scenario, ruleApplier))))); rowValues.add(new ResultValue(util.convertNull(String.valueOf(getResultRowCount())))); return rowValues; } - private String getExplainPlan() { + private String getExplainPlan(Scenario scenario, RulesApplier ruleApplier) { try { - return pUtil.getExplainPlan(this); + return pUtil.getExplainPlan(this, scenario, ruleApplier); } catch (SQLException e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java index 5e0f242..929f96a 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java @@ -22,6 +22,7 @@ import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.result.file.ResultFileDetails; import org.apache.phoenix.pherf.result.impl.CSVFileResultHandler; import org.apache.phoenix.pherf.result.impl.XMLResultHandler; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.apache.phoenix.util.InstanceResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,18 +82,23 @@ public class ResultManager { } } + + public synchronized void write(DataModelResult result) throws Exception { + write(result, null); + } + /** * Write out the result to each writer in the pool * * @param result {@link DataModelResult} * @throws Exception */ - public synchronized void write(DataModelResult result) throws Exception { + public synchronized void write(DataModelResult result, RulesApplier ruleApplier) throws Exception { try { util.ensureBaseResultDirExists(); final DataModelResult dataModelResultCopy = new DataModelResult(result); for (ResultHandler handler : resultHandlers) { - util.write(handler, dataModelResultCopy); + util.write(handler, dataModelResultCopy, ruleApplier); } } finally { for (ResultHandler handler : resultHandlers) { @@ -108,13 +114,17 @@ public class ResultManager { } } + public synchronized void write(List<DataModelResult> dataModelResults) throws Exception { + write(dataModelResults, null); + } + /** * Write a combined set of results for each result in the list. * * @param dataModelResults List<{@link DataModelResult > </>} * @throws Exception */ - public synchronized void write(List<DataModelResult> dataModelResults) throws Exception { + public synchronized void write(List<DataModelResult> dataModelResults, RulesApplier rulesApplier) throws Exception { util.ensureBaseResultDirExists(); CSVFileResultHandler detailsCSVWriter = null; @@ -123,7 +133,7 @@ public class ResultManager { detailsCSVWriter.setResultFileDetails(ResultFileDetails.CSV_DETAILED_PERFORMANCE); detailsCSVWriter.setResultFileName(PherfConstants.COMBINED_FILE_NAME); for (DataModelResult dataModelResult : dataModelResults) { - util.write(detailsCSVWriter, dataModelResult); + util.write(detailsCSVWriter, dataModelResult, rulesApplier); } } finally { if (detailsCSVWriter != null) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java index 0c2a7b8..30988ef 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultUtil.java @@ -23,6 +23,7 @@ import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.result.file.ResultFileDetails; import org.apache.phoenix.pherf.result.impl.CSVFileResultHandler; import org.apache.phoenix.pherf.result.impl.CSVResultHandler; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.apache.phoenix.pherf.util.PhoenixUtil; import java.io.File; @@ -117,7 +118,7 @@ public class ResultUtil { } } - public synchronized void write(ResultHandler resultHandler, DataModelResult dataModelResult) + public synchronized void write(ResultHandler resultHandler, DataModelResult dataModelResult, RulesApplier ruleApplier) throws Exception { ResultFileDetails resultFileDetails = resultHandler.getResultFileDetails(); switch (resultFileDetails) { @@ -126,7 +127,7 @@ public class ResultUtil { case CSV_DETAILED_FUNCTIONAL: List<List<ResultValue>> rowDetails = - getCSVResults(dataModelResult, resultFileDetails); + getCSVResults(dataModelResult, resultFileDetails, ruleApplier); for (List<ResultValue> row : rowDetails) { Result result = @@ -199,7 +200,7 @@ public class ResultUtil { } private List<List<ResultValue>> getCSVResults(DataModelResult dataModelResult, - ResultFileDetails resultFileDetails) { + ResultFileDetails resultFileDetails, RulesApplier ruleApplier) { List<List<ResultValue>> rowList = new ArrayList<>(); for (ScenarioResult result : dataModelResult.getScenarioResult()) { @@ -207,7 +208,7 @@ public class ResultUtil { for (QueryResult queryResult : querySetResult.getQueryResults()) { switch (resultFileDetails) { case CSV_AGGREGATE_PERFORMANCE: - List<ResultValue> csvResult = queryResult.getCsvRepresentation(this); + List<ResultValue> csvResult = queryResult.getCsvRepresentation(this, result, ruleApplier); rowList.add(csvResult); break; case CSV_DETAILED_PERFORMANCE: http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java index 454050b..2afc29a 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java @@ -39,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong; public class RulesApplier { private static final Logger logger = LoggerFactory.getLogger(RulesApplier.class); - private static final AtomicLong COUNTER = new AtomicLong(100); + private static final AtomicLong COUNTER = new AtomicLong(0); // Used to bail out of random distribution if it takes too long // This should never happen when distributions add up to 100 @@ -51,6 +51,9 @@ public class RulesApplier { private final XMLConfigParser parser; private final List<Map> modelList; + private final Map<String, Column> columnMap; + private String cachedScenarioOverrideName; + private Map<DataTypeMapping, List> scenarioOverrideMap; public RulesApplier(XMLConfigParser parser) { @@ -60,15 +63,39 @@ public class RulesApplier { public RulesApplier(XMLConfigParser parser, long seed) { this.parser = parser; this.modelList = new ArrayList<Map>(); + this.columnMap = new HashMap<String, Column>(); this.rndNull = new Random(seed); this.rndVal = new Random(seed); this.randomDataGenerator = new RandomDataGenerator(); + this.cachedScenarioOverrideName = null; populateModelList(); } public List<Map> getModelList() { return Collections.unmodifiableList(this.modelList); } + + private Map<DataTypeMapping, List> getCachedScenarioOverrides(Scenario scenario) { + if (this.cachedScenarioOverrideName == null || this.cachedScenarioOverrideName != scenario.getName()) { + this.cachedScenarioOverrideName = scenario.getName(); + this.scenarioOverrideMap = new HashMap<DataTypeMapping, List>(); + + if (scenario.getDataOverride() != null) { + for (Column column : scenario.getDataOverride().getColumn()) { + List<Column> cols; + DataTypeMapping type = column.getType(); + if (this.scenarioOverrideMap.containsKey(type)) { + this.scenarioOverrideMap.get(type).add(column); + } else { + cols = new LinkedList<Column>(); + cols.add(column); + this.scenarioOverrideMap.put(type, cols); + } + } + } + } + return scenarioOverrideMap; + } /** @@ -84,11 +111,26 @@ public class RulesApplier { */ public DataValue getDataForRule(Scenario scenario, Column phxMetaColumn) throws Exception { // TODO Make a Set of Rules that have already been applied so that so we don't generate for every value - + List<Scenario> scenarios = parser.getScenarios(); DataValue value = null; if (scenarios.contains(scenario)) { logger.debug("We found a correct Scenario"); + + Map<DataTypeMapping, List> overrideRuleMap = this.getCachedScenarioOverrides(scenario); + + if (overrideRuleMap != null) { + List<Column> overrideRuleList = this.getCachedScenarioOverrides(scenario).get(phxMetaColumn.getType()); + + if (overrideRuleList != null && overrideRuleList.contains(phxMetaColumn)) { + logger.debug("We found a correct override column rule"); + Column columnRule = getColumnForRuleOverride(overrideRuleList, phxMetaColumn); + if (columnRule != null) { + return getDataValue(columnRule); + } + } + } + // Assume the first rule map Map<DataTypeMapping, List> ruleMap = modelList.get(0); List<Column> ruleList = ruleMap.get(phxMetaColumn.getType()); @@ -107,6 +149,7 @@ public class RulesApplier { } } + return value; } @@ -140,19 +183,9 @@ public class RulesApplier { switch (column.getType()) { case VARCHAR: - // Use the specified data values from configs if they exist - if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = pickDataValueFromList(dataValues); - } else { - Preconditions.checkArgument(length > 0, "length needs to be > 0"); - if (column.getDataSequence() == DataSequence.SEQUENTIAL) { - data = getSequentialDataValue(column); - } else { - data = getRandomDataValue(column); - } - } - break; + case VARBINARY: case CHAR: + // Use the specified data values from configs if they exist if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { data = pickDataValueFromList(dataValues); } else { @@ -164,6 +197,17 @@ public class RulesApplier { } } break; + case VARCHAR_ARRAY: + //only list datavalues are supported + String arr = ""; + for (DataValue dv : dataValues) { + arr += "," + dv.getValue(); + } + if (arr.startsWith(",")) { + arr = arr.replaceFirst(",", ""); + } + data = new DataValue(column.getType(), arr); + break; case DECIMAL: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { data = pickDataValueFromList(dataValues); @@ -171,8 +215,8 @@ public class RulesApplier { int precision = column.getPrecision(); double minDbl = column.getMinValue(); Preconditions.checkArgument((precision > 0) && (precision <= 18), "Precision must be between 0 and 18"); - Preconditions.checkArgument(minDbl >= 0, "minvalue must be set in configuration"); - Preconditions.checkArgument(column.getMaxValue() > 0, "maxValue must be set in configuration"); + Preconditions.checkArgument(minDbl >= 0, "minvalue must be set in configuration for decimal"); + Preconditions.checkArgument(column.getMaxValue() > 0, "maxValue must be set in configuration decimal"); StringBuilder maxValueStr = new StringBuilder(); for (int i = 0; i < precision; i++) { @@ -188,22 +232,34 @@ public class RulesApplier { if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { data = pickDataValueFromList(dataValues); } else { - int minInt = column.getMinValue(); - int maxInt = column.getMaxValue(); - Preconditions.checkArgument((minInt > 0) && (maxInt > 0), "min and max values need to be set in configuration"); + int minInt = (int) column.getMinValue(); + int maxInt = (int) column.getMaxValue(); + Preconditions.checkArgument((minInt > 0) && (maxInt > 0), "min and max values need to be set in configuration for integers " + column.getName()); int intVal = RandomUtils.nextInt(minInt, maxInt); data = new DataValue(column.getType(), String.valueOf(intVal)); } break; + case UNSIGNED_LONG: + if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { + data = pickDataValueFromList(dataValues); + } else { + long minLong = column.getMinValue(); + long maxLong = column.getMaxValue(); + Preconditions.checkArgument((minLong > 0) && (maxLong > 0), "min and max values need to be set in configuration for unsigned_longs " + column.getName()); + long longVal = RandomUtils.nextLong(minLong, maxLong); + data = new DataValue(column.getType(), String.valueOf(longVal)); + } + break; case DATE: + case TIMESTAMP: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { data = pickDataValueFromList(dataValues); // Check if date has right format or not data.setValue(checkDatePattern(data.getValue())); } else if (column.getUseCurrentDate() != true){ - int minYear = column.getMinValue(); - int maxYear = column.getMaxValue(); - Preconditions.checkArgument((minYear > 0) && (maxYear > 0), "min and max values need to be set in configuration"); + int minYear = (int) column.getMinValue(); + int maxYear = (int) column.getMaxValue(); + Preconditions.checkArgument((minYear > 0) && (maxYear > 0), "min and max values need to be set in configuration for date/timestamps " + column.getName()); String dt = generateRandomDate(minYear, maxYear); data = new DataValue(column.getType(), dt); @@ -353,13 +409,15 @@ public class RulesApplier { if (!modelList.isEmpty()) { return; } - + // Support for multiple models, but rules are only relevant each model for (DataModel model : parser.getDataModels()) { // Step 1 final Map<DataTypeMapping, List> ruleMap = new HashMap<DataTypeMapping, List>(); for (Column column : model.getDataMappingColumns()) { + columnMap.put(column.getName(), column); + List<Column> cols; DataTypeMapping type = column.getType(); if (ruleMap.containsKey(type)) { @@ -382,7 +440,33 @@ public class RulesApplier { List<Column> ruleList = ruleMap.get(phxMetaColumn.getType()); return getColumnForRule(ruleList, phxMetaColumn); } + + public Column getRule(String columnName) { + return getRule(columnName, null); + } + + public Column getRule(String columnName, Scenario scenario) { + if (null != scenario && null != scenario.getDataOverride()) { + for (Column column: scenario.getDataOverride().getColumn()) { + if (column.getName().equals(columnName)) { + return column; + } + } + } + + return columnMap.get(columnName); + } + private Column getColumnForRuleOverride(List<Column> ruleList, Column phxMetaColumn) { + for (Column columnRule : ruleList) { + if (columnRule.getName().equals(phxMetaColumn.getName())) { + return new Column(columnRule); + } + } + + return null; + } + private Column getColumnForRule(List<Column> ruleList, Column phxMetaColumn) { // Column pointer to head of list @@ -400,7 +484,7 @@ public class RulesApplier { ruleAppliedColumn.mutate(columnRule); } - return ruleAppliedColumn; + return ruleAppliedColumn; } /** @@ -414,10 +498,12 @@ public class RulesApplier { DataValue data = null; long inc = COUNTER.getAndIncrement(); String strInc = String.valueOf(inc); - String varchar = RandomStringUtils.randomAlphanumeric(column.getLength() - strInc.length()); - varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc + varchar : - strInc + varchar; - + int paddedLength = column.getLengthExcludingPrefix(); + String strInc1 = StringUtils.leftPad(strInc, paddedLength, "0"); + String strInc2 = StringUtils.right(strInc1, column.getLengthExcludingPrefix()); + String varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc2: + strInc2; + // Truncate string back down if it exceeds length varchar = StringUtils.left(varchar,column.getLength()); data = new DataValue(column.getType(), varchar); http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/schema/SchemaReader.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/schema/SchemaReader.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/schema/SchemaReader.java index 439f87e..5ccdaaa 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/schema/SchemaReader.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/schema/SchemaReader.java @@ -62,7 +62,7 @@ public class SchemaReader { public void applySchema() throws Exception { Connection connection = null; try { - connection = pUtil.getConnection(); + connection = pUtil.getConnection(null); for (Path file : resourceList) { logger.info("\nApplying schema to file: " + file); pUtil.executeStatement(resourceToString(file), connection); http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java index 38dcd64..72ab3e0 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java @@ -25,6 +25,8 @@ import org.apache.phoenix.pherf.configuration.*; import org.apache.phoenix.pherf.jmx.MonitorManager; import org.apache.phoenix.pherf.result.DataLoadThreadTime; import org.apache.phoenix.pherf.result.DataLoadTimeSummary; +import org.apache.phoenix.pherf.rules.RulesApplier; +import org.apache.phoenix.pherf.util.GoogleChartGenerator.Node; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,7 @@ import java.sql.*; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Properties; import java.util.Set; @@ -50,6 +53,7 @@ public class PhoenixUtil { private static String queryServerUrl; private static final String ASYNC_KEYWORD = "ASYNC"; private static final int ONE_MIN_IN_MS = 60000; + private static String CurrentSCN = null; private PhoenixUtil() { this(false); @@ -86,10 +90,14 @@ public class PhoenixUtil { } public Connection getConnection(String tenantId) throws Exception { - return getConnection(tenantId, testEnabled); + return getConnection(tenantId, testEnabled, null); + } + + public Connection getConnection(String tenantId, Map<String, String> phoenixProperty) throws Exception { + return getConnection(tenantId, testEnabled, phoenixProperty); } - private Connection getConnection(String tenantId, boolean testEnabled) throws Exception { + public Connection getConnection(String tenantId, boolean testEnabled, Map<String, String> phoenixProperty) throws Exception { if (useThinDriver) { if (null == queryServerUrl) { throw new IllegalArgumentException("QueryServer URL must be set before" + @@ -112,6 +120,16 @@ public class PhoenixUtil { props.setProperty("TenantId", tenantId); logger.debug("\nSetting tenantId to " + tenantId); } + + if (phoenixProperty != null) { + for (Map.Entry<String, String> phxProperty: phoenixProperty.entrySet()) { + props.setProperty(phxProperty.getKey(), phxProperty.getValue()); + System.out.println("Setting connection property " + + phxProperty.getKey() + " to " + + phxProperty.getValue()); + } + } + String url = "jdbc:phoenix:" + zookeeper + (testEnabled ? ";test=true" : ""); return DriverManager.getConnection(url, props); } @@ -232,7 +250,7 @@ public class PhoenixUtil { public ResultSet getColumnsMetaData(String schemaName, String tableName, Connection connection) throws SQLException { DatabaseMetaData dbmd = connection.getMetaData(); - ResultSet resultSet = dbmd.getColumns(null, schemaName, tableName, null); + ResultSet resultSet = dbmd.getColumns(null, schemaName.toUpperCase(), tableName.toUpperCase(), null); return resultSet; } @@ -245,7 +263,7 @@ public class PhoenixUtil { while (resultSet.next()) { Column column = new Column(); column.setName(resultSet.getString("COLUMN_NAME")); - column.setType(DataTypeMapping.valueOf(resultSet.getString("TYPE_NAME"))); + column.setType(DataTypeMapping.valueOf(resultSet.getString("TYPE_NAME").replace(" ", "_"))); column.setLength(resultSet.getInt("COLUMN_SIZE")); columnList.add(column); } @@ -392,21 +410,35 @@ public class PhoenixUtil { executeStatement("UPDATE STATISTICS " + tableName, scenario); } + public String getExplainPlan(Query query) throws SQLException { + return getExplainPlan(query, null, null); + } + /** * Get explain plan for a query * * @param query + * @param ruleApplier + * @param scenario * @return * @throws SQLException */ - public String getExplainPlan(Query query) throws SQLException { + public String getExplainPlan(Query query, Scenario scenario, RulesApplier ruleApplier) throws SQLException { Connection conn = null; ResultSet rs = null; PreparedStatement statement = null; StringBuilder buf = new StringBuilder(); try { conn = getConnection(query.getTenantId()); - statement = conn.prepareStatement("EXPLAIN " + query.getStatement()); + String explainQuery; + if (scenario != null && ruleApplier != null) { + explainQuery = query.getDynamicStatement(ruleApplier, scenario); + } + else { + explainQuery = query.getStatement(); + } + + statement = conn.prepareStatement("EXPLAIN " + explainQuery); rs = statement.executeQuery(); while (rs.next()) { buf.append(rs.getString(1).trim().replace(",", "-")); http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java index 24c68dc..7b9313f 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java @@ -28,10 +28,14 @@ import org.apache.phoenix.pherf.result.DataModelResult; import org.apache.phoenix.pherf.result.ResultManager; import org.apache.phoenix.pherf.result.RunTime; import org.apache.phoenix.pherf.result.ThreadTime; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Query; +import org.apache.phoenix.pherf.configuration.Scenario; +import org.apache.phoenix.pherf.configuration.WriteParams; +import org.apache.phoenix.pherf.configuration.XMLConfigParser; import org.apache.phoenix.pherf.util.PhoenixUtil; class MultiThreadedRunner implements Runnable { @@ -45,6 +49,11 @@ class MultiThreadedRunner implements Runnable { private long executionDurationInMs; private static long lastResultWritten = System.currentTimeMillis() - 1000; private final ResultManager resultManager; + private final RulesApplier ruleApplier; + private final Scenario scenario; + private final WorkloadExecutor workloadExecutor; + private final XMLConfigParser parser; + /** * MultiThreadedRunner @@ -55,16 +64,21 @@ class MultiThreadedRunner implements Runnable { * @param threadTime * @param numberOfExecutions * @param executionDurationInMs + * @param ruleRunner */ MultiThreadedRunner(String threadName, Query query, DataModelResult dataModelResult, - ThreadTime threadTime, long numberOfExecutions, long executionDurationInMs, boolean writeRuntimeResults) { + ThreadTime threadTime, long numberOfExecutions, long executionDurationInMs, boolean writeRuntimeResults, RulesApplier ruleApplier, Scenario scenario, WorkloadExecutor workloadExecutor, XMLConfigParser parser) { this.query = query; this.threadName = threadName; this.threadTime = threadTime; this.dataModelResult = dataModelResult; this.numberOfExecutions = numberOfExecutions; this.executionDurationInMs = executionDurationInMs; + this.ruleApplier = ruleApplier; + this.scenario = scenario; this.resultManager = new ResultManager(dataModelResult.getName(), writeRuntimeResults); + this.workloadExecutor = workloadExecutor; + this.parser = parser; } /** @@ -81,7 +95,7 @@ class MultiThreadedRunner implements Runnable { synchronized (resultManager) { timedQuery(); if ((System.currentTimeMillis() - lastResultWritten) > 1000) { - resultManager.write(dataModelResult); + resultManager.write(dataModelResult, ruleApplier); lastResultWritten = System.currentTimeMillis(); } } @@ -108,7 +122,7 @@ class MultiThreadedRunner implements Runnable { private void timedQuery() throws Exception { boolean isSelectCountStatement = - query.getStatement().toUpperCase().trim().contains("COUNT(*)") ? true : false; + query.getStatement().toUpperCase().trim().contains("COUNT(") ? true : false; Connection conn = null; PreparedStatement statement = null; @@ -119,8 +133,17 @@ class MultiThreadedRunner implements Runnable { long resultRowCount = 0; try { - conn = pUtil.getConnection(query.getTenantId()); - statement = conn.prepareStatement(query.getStatement()); + conn = pUtil.getConnection(query.getTenantId(), scenario.getPhoenixProperties()); + conn.setAutoCommit(true); + final String statementString = query.getDynamicStatement(ruleApplier, scenario); + statement = conn.prepareStatement(statementString); + logger.info("Executing: " + statementString); + + if (scenario.getWriteParams() != null) { + Workload writes = new WriteWorkload(PhoenixUtil.create(), parser, scenario, GeneratePhoenixStats.NO); + workloadExecutor.add(writes); + } + boolean isQuery = statement.execute(); if (isQuery) { rs = statement.getResultSet(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java index 7f861f1..8d0ced5 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.*; import org.apache.phoenix.pherf.result.*; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +43,7 @@ public class QueryExecutor implements Workload { private final PhoenixUtil util; private final WorkloadExecutor workloadExecutor; private final boolean writeRuntimeResults; + private RulesApplier ruleApplier; public QueryExecutor(XMLConfigParser parser, PhoenixUtil util, WorkloadExecutor workloadExecutor) { @@ -64,6 +66,7 @@ public class QueryExecutor implements Workload { this.util = util; this.workloadExecutor = workloadExecutor; this.writeRuntimeResults = writeRuntimeResults; + this.ruleApplier = new RulesApplier(parser); } @Override @@ -143,17 +146,6 @@ public class QueryExecutor implements Workload { ScenarioResult scenarioResult = new ScenarioResult(scenario); scenarioResult.setPhoenixProperties(phoenixProperty); dataModelResult.getScenarioResult().add(scenarioResult); - WriteParams writeParams = scenario.getWriteParams(); - - if (writeParams != null) { - int writerThreadCount = writeParams.getWriterThreadCount(); - for (int i = 0; i < writerThreadCount; i++) { - logger.debug("Inserting write workload ( " + i + " ) of ( " - + writerThreadCount + " )"); - Workload writes = new WriteWorkload(PhoenixUtil.create(), parser, GeneratePhoenixStats.NO); - workloadExecutor.add(writes); - } - } for (QuerySet querySet : scenario.getQuerySet()) { QuerySetResult querySetResult = new QuerySetResult(querySet); @@ -161,14 +153,14 @@ public class QueryExecutor implements Workload { util.executeQuerySetDdls(querySet); if (querySet.getExecutionType() == ExecutionType.SERIAL) { - executeQuerySetSerial(dataModelResult, querySet, querySetResult); + executeQuerySetSerial(dataModelResult, querySet, querySetResult, scenario); } else { - executeQuerySetParallel(dataModelResult, querySet, querySetResult); + executeQuerySetParallel(dataModelResult, querySet, querySetResult, scenario); } } - resultManager.write(dataModelResult); + resultManager.write(dataModelResult, ruleApplier); } - resultManager.write(dataModelResults); + resultManager.write(dataModelResults, ruleApplier); resultManager.flush(); } catch (Exception e) { logger.warn("", e); @@ -183,10 +175,11 @@ public class QueryExecutor implements Workload { * @param dataModelResult * @param querySet * @param querySetResult + * @param scenario * @throws InterruptedException */ protected void executeQuerySetSerial(DataModelResult dataModelResult, QuerySet querySet, - QuerySetResult querySetResult) throws InterruptedException { + QuerySetResult querySetResult, Scenario scenario) throws InterruptedException { for (Query query : querySet.getQuery()) { QueryResult queryResult = new QueryResult(query); querySetResult.getQueryResults().add(queryResult); @@ -200,7 +193,7 @@ public class QueryExecutor implements Workload { Runnable thread = executeRunner((i + 1) + "," + cr, dataModelResult, queryResult, - querySetResult); + querySetResult, scenario); threads.add(workloadExecutor.getPool().submit(thread)); } @@ -224,7 +217,7 @@ public class QueryExecutor implements Workload { * @throws InterruptedException */ protected void executeQuerySetParallel(DataModelResult dataModelResult, QuerySet querySet, - QuerySetResult querySetResult) throws InterruptedException { + QuerySetResult querySetResult, Scenario scenario) throws InterruptedException { for (int cr = querySet.getMinConcurrency(); cr <= querySet.getMaxConcurrency(); cr++) { List<Future> threads = new ArrayList<>(); for (int i = 0; i < cr; i++) { @@ -235,7 +228,7 @@ public class QueryExecutor implements Workload { Runnable thread = executeRunner((i + 1) + "," + cr, dataModelResult, queryResult, - querySetResult); + querySetResult, scenario); threads.add(workloadExecutor.getPool().submit(thread)); } @@ -257,10 +250,11 @@ public class QueryExecutor implements Workload { * @param dataModelResult * @param queryResult * @param querySet + * @param scenario * @return */ protected Runnable executeRunner(String name, DataModelResult dataModelResult, - QueryResult queryResult, QuerySet querySet) { + QueryResult queryResult, QuerySet querySet, Scenario scenario) { ThreadTime threadTime = new ThreadTime(); queryResult.getThreadTimes().add(threadTime); threadTime.setThreadName(name); @@ -271,7 +265,7 @@ public class QueryExecutor implements Workload { thread = new MultiThreadedRunner(threadTime.getThreadName(), queryResult, dataModelResult, threadTime, querySet.getNumberOfExecutions(), - querySet.getExecutionDurationInMs(), writeRuntimeResults); + querySet.getExecutionDurationInMs(), writeRuntimeResults, ruleApplier, scenario, workloadExecutor, parser); } else { thread = new MultithreadedDiffer(threadTime.getThreadName(), queryResult, threadTime, http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java index 3574761..4023383 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java @@ -19,10 +19,12 @@ package org.apache.phoenix.pherf.workload; import java.math.BigDecimal; +import java.sql.Array; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Timestamp; import java.sql.Types; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -33,6 +35,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Column; @@ -108,21 +111,27 @@ public class WriteWorkload implements Workload { this.rulesApplier = new RulesApplier(parser); this.resultUtil = new ResultUtil(); this.generateStatistics = generateStatistics; - + int size = Integer.parseInt(properties.getProperty("pherf.default.dataloader.threadpool")); + // Overwrite defaults properties with those given in the configuration. This indicates the // scenario is a R/W mixed workload. if (scenario != null) { this.scenario = scenario; writeParams = scenario.getWriteParams(); - threadSleepDuration = writeParams.getThreadSleepDuration(); + if (writeParams != null) { + threadSleepDuration = writeParams.getThreadSleepDuration(); + size = writeParams.getWriterThreadCount(); + } + else { + threadSleepDuration = 0; + } + } else { writeParams = null; this.scenario = null; threadSleepDuration = 0; } - int size = Integer.parseInt(properties.getProperty("pherf.default.dataloader.threadpool")); - // Should addBatch/executeBatch be used? Default: false this.useBatchApi = Boolean.getBoolean(USE_BATCH_API_PROPERTY); @@ -381,17 +390,50 @@ public class WriteWorkload implements Workload { statement.setInt(count, Integer.parseInt(dataValue.getValue())); } break; + case UNSIGNED_LONG: + if (dataValue.getValue().equals("")) { + statement.setNull(count, Types.LONGVARCHAR); + } else { + statement.setLong(count, Long.parseLong(dataValue.getValue())); + } + break; case DATE: if (dataValue.getValue().equals("")) { statement.setNull(count, Types.DATE); } else { Date date = - new java.sql.Date( - simpleDateFormat.parse(dataValue.getValue()).getTime()); + new java.sql.Date(simpleDateFormat.parse(dataValue.getValue()).getTime()); statement.setDate(count, date); } break; + case VARCHAR_ARRAY: + if (dataValue.getValue().equals("")) { + statement.setNull(count, Types.ARRAY); + } else { + Array + arr = + statement.getConnection().createArrayOf("VARCHAR", dataValue.getValue().split(",")); + statement.setArray(count, arr); + } + break; + case VARBINARY: + if (dataValue.getValue().equals("")) { + statement.setNull(count, Types.VARBINARY); + } else { + statement.setBytes(count, dataValue.getValue().getBytes()); + } + break; + case TIMESTAMP: + if (dataValue.getValue().equals("")) { + statement.setNull(count, Types.TIMESTAMP); + } else { + java.sql.Timestamp + ts = + new java.sql.Timestamp(simpleDateFormat.parse(dataValue.getValue()).getTime()); + statement.setTimestamp(count, ts); + } + break; default: break; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/main/resources/scenario/prod_test_unsalted_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/resources/scenario/prod_test_unsalted_scenario.xml b/phoenix-pherf/src/main/resources/scenario/prod_test_unsalted_scenario.xml index e538ac2..1c32b75 100644 --- a/phoenix-pherf/src/main/resources/scenario/prod_test_unsalted_scenario.xml +++ b/phoenix-pherf/src/main/resources/scenario/prod_test_unsalted_scenario.xml @@ -349,8 +349,6 @@ </scenario> <scenario tableName="PHERF.PHERF_PROD_TEST_UNSALTED" rowCount="10"> - <!-- Scenario level rule overrides will be unsupported in V1. - You can use the general datamappings in the mean time--> <dataOverride> <column> <type>VARCHAR</type> @@ -370,8 +368,20 @@ <name>TENANT_ID</name> </column> </dataOverride> + + <!-- Pre and post scenario indexes --> + <preScenarioDdls> + <ddl>CREATE INDEX IDX_DIVISION ON PHERF.PHERF_PROD_TEST_UNSALTED (DIVISION)</ddl> + </preScenarioDdls> + + <postScenarioDdls> + <ddl>CREATE INDEX IDX_OLDVAL_STRING ON PHERF.PHERF_PROD_TEST_UNSALTED (OLDVAL_STRING)</ddl> + <ddl>CREATE INDEX IDX_CONNECTION_ID ON PHERF.PHERF_PROD_TEST_UNSALTED (CONNECTION_ID)</ddl> + </postScenarioDdls> + <!--Minimum of executionDurationInMs or numberOfExecutions. Which ever is reached first --> <querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="60000" numberOfExecutions="100"> + <query statement="select count(*) from PHERF.PHERF_PROD_TEST_UNSALTED WHERE TENANT_ID=[TENANT_ID] AND TENANT_ID=[TENANT_ID]"/> <!-- Aggregate queries on a per tenant basis --> <query tenantId="00Dxx0000001gER" ddl="CREATE VIEW IF NOT EXISTS PHERF.PHERF_TEST_VIEW_UNSALTED AS SELECT * FROM PHERF.PHERF_PROD_TEST_UNSALTED" http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ColumnTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ColumnTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ColumnTest.java index e573c07..35e8754 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ColumnTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ColumnTest.java @@ -18,6 +18,9 @@ package org.apache.phoenix.pherf; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataTypeMapping; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java index 1f1006d..5afde69 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java @@ -71,8 +71,8 @@ public class ConfigurationParserTest extends ResultBaseTest { assertTrue("Could not load the data columns from xml.", (dataMappingColumns != null) && (dataMappingColumns.size() > 0)); assertTrue("Could not load the data DataValue list from xml.", - (dataMappingColumns.get(6).getDataValues() != null) - && (dataMappingColumns.get(6).getDataValues().size() > 0)); + (dataMappingColumns.get(8).getDataValues() != null) + && (dataMappingColumns.get(8).getDataValues().size() > 0)); assertDateValue(dataMappingColumns); assertCurrentDateValue(dataMappingColumns); http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index 228cd58..f4b0e5c 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -24,8 +24,10 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -245,6 +247,87 @@ public class RuleGeneratorTest { testSet.size() == (threadCount * increments)); } + @Test + public void testTimestampRule() throws Exception { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat df = new SimpleDateFormat("yyyy"); + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + Scenario scenario = parser.getScenarios().get(0); + + Column simPhxCol = new Column(); + simPhxCol.setName("TS_DATE"); + simPhxCol.setType(DataTypeMapping.TIMESTAMP); + + // Run this 10 times gives a reasonable chance that all the values will appear at least once + for (int i = 0; i < 10; i++) { + DataValue value = rulesApplier.getDataForRule(scenario, simPhxCol); + Date dt = simpleDateFormat.parse(value.getValue()); + int year = Integer.parseInt(df.format(dt)); + assertTrue("Got unexpected TS value" + value.getValue(), year >= 2020 && year <= 2025); + } + } + + @Test + public void testVarcharArray() throws Exception { + + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + + // Run this 15 times gives a reasonable chance that all the values will appear at least once + for (int i = 0; i < 15; i++) { + Column c = rulesApplier.getRule("VAR_ARRAY"); + DataValue value = rulesApplier.getDataValue(c); + assertTrue("Got a value not in the list for the rule. :" + value.getValue(), value.getValue().equals("Foo,Bar")); + } + } + + @Test + public void testVarBinary() throws Exception { + List<String> expectedValues = new ArrayList(); + for (int i=0; i<10; i++) { + expectedValues.add("VBOxx00" + i); + } + + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + + for (int i = 0; i < 5; i++) { + Column c = rulesApplier.getRule("VAR_BIN"); + DataValue value = rulesApplier.getDataValue(c); + System.out.println(value.getValue()); + assertTrue("Got a value not in the list for the rule. :" + value.getValue(), expectedValues.contains(value.getValue())); + } + } + + @Test + public void testPrefixSequence() throws Exception { + List<String> expectedValues = new ArrayList(); + expectedValues.add("0F90000000000X0"); + expectedValues.add("0F90000000000X1"); + expectedValues.add("0F90000000000X2"); + expectedValues.add("0F90000000000X3"); + expectedValues.add("0F90000000000X4"); + expectedValues.add("0F90000000000X5"); + expectedValues.add("0F90000000000X6"); + expectedValues.add("0F90000000000X7"); + expectedValues.add("0F90000000000X8"); + expectedValues.add("0F90000000000X9"); + + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + + // Run this 15 times gives a reasonable chance that all the values will appear at least once + for (int i = 0; i < 15; i++) { + DataValue value = rulesApplier.getDataValue(rulesApplier.getRule("NEWVAL_STRING")); + assertTrue("Got a value not in the list for the rule. :" + value.getValue(), expectedValues.contains(value.getValue())); + } + } + @Test public void testValueListRule() throws Exception { List<String> expectedValues = new ArrayList(); @@ -301,12 +384,30 @@ public class RuleGeneratorTest { assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType()); assertEquals("Rule contains incorrect length.", rule.getLength(), 10); assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "MYPRFX"); - + value = rulesApplier.getDataForRule(scenario, simPhxCol); - assertEquals("Value returned does not match rule.", value.getValue().length(), 10); - assertTrue("Value returned start with prefix.", + assertEquals("Value returned does not match rule.", 10, value.getValue().length()); + assertTrue("Value returned start with prefix. " + value.getValue(), StringUtils.startsWith(value.getValue(), rule.getPrefix())); + + } + + + @Test + public void testScenarioLevelRuleOverride() throws Exception { + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + Scenario scenario = parser.getScenarios().get(0); + + // Test scenario level overridden rule + Column simPhxCol = new Column(); + simPhxCol.setName("FIELD"); + simPhxCol.setType(DataTypeMapping.VARCHAR); + DataValue value = rulesApplier.getDataForRule(scenario, simPhxCol); + assertEquals("Override rule should contain field length of 5", 5, value.getValue().length()); } + /** * Asserts that the value field is between the min/max value fields http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/test/resources/datamodel/test_schema.sql ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql index 21034d9..fa9952b 100644 --- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql @@ -20,9 +20,13 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE ( PARENT_ID CHAR(15) NOT NULL, CREATED_DATE DATE NOT NULL, NOW_DATE DATE, + TS_DATE TIMESTAMP, PRESENT_DATE DATE, OTHER_ID CHAR(15), FIELD VARCHAR, + VAR_ARRAY VARCHAR ARRAY, + VAR_BIN VARBINARY, + DIVISION INTEGER, OLDVAL_STRING VARCHAR, NEWVAL_STRING VARCHAR, SOME_INT INTEGER http://git-wip-us.apache.org/repos/asf/phoenix/blob/073392e3/phoenix-pherf/src/test/resources/scenario/test_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index e2915ba..99ce81b 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -33,6 +33,18 @@ <name>GENERAL_CHAR</name> </column> <column> + <type>TIMESTAMP</type> + <!--SEQUENTIAL is unsupported for DATE --> + <dataSequence>RANDOM</dataSequence> + <!-- Number [0-100] that represents the probability of creating a null value --> + <!-- The higher the number, the more like the value will returned will be null --> + <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null --> + <nullChance>0</nullChance> + <minValue>2020</minValue> + <maxValue>2025</maxValue> + <name>GENERAL_TIMESTAMP</name> + </column> + <column> <type>DATE</type> <!--SEQUENTIAL is unsupported for DATE --> <dataSequence>RANDOM</dataSequence> @@ -150,9 +162,22 @@ <type>VARCHAR</type> <length>15</length> <userDefined>true</userDefined> - <dataSequence>RANDOM</dataSequence> + <dataSequence>SEQUENTIAL</dataSequence> <name>NEWVAL_STRING</name> - <prefix>TSTPRFX</prefix> + <prefix>0F90000000000X</prefix> + </column> + <column> + <type>VARCHAR_ARRAY</type> + <userDefined>true</userDefined> + <name>VAR_ARRAY</name> + <valuelist> + <datavalue> + <value>Foo</value> + </datavalue> + <datavalue> + <value>Bar</value> + </datavalue> + </valuelist> </column> <column> <type>CHAR</type> @@ -181,6 +206,21 @@ <name>OTHER_ID</name> <prefix>z0Oxx00</prefix> </column> + <column> + <type>VARBINARY</type> + <userDefined>true</userDefined> + <dataSequence>SEQUENTIAL</dataSequence> + <length>8</length> + <name>VAR_BIN</name> + <prefix>VBOxx00</prefix> + </column> + <column> + <type>VARCHAR</type> + <userDefined>true</userDefined> + <dataSequence>SEQUENTIAL</dataSequence> + <length>1</length> + <name>FIELD</name> + </column> </datamapping> <scenarios> <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW"> @@ -191,7 +231,7 @@ <type>VARCHAR</type> <userDefined>true</userDefined> <dataSequence>RANDOM</dataSequence> - <length>10</length> + <length>5</length> <name>FIELD</name> </column> </dataOverride> @@ -219,7 +259,7 @@ </writeParams> <querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="10000"> <query id="q3" statement="select count(*) from PHERF.TEST_TABLE"/> - <query id="q4" statement="select sum(DIVISION) from PHERF.TEST_TABLE"/> + <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/> </querySet> </scenario>