PHOENIX-2337 - Pherf - Prefix values override rule values for writes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/caa0dbb4 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/caa0dbb4 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/caa0dbb4 Branch: refs/heads/txn Commit: caa0dbb46a47945eaa384b31f7a2ba8e9c7a6c08 Parents: 28007f8 Author: Cody Marcel <[email protected]> Authored: Mon Oct 19 13:25:15 2015 -0700 Committer: Cody Marcel <[email protected]> Committed: Tue Oct 20 13:55:12 2015 -0700 ---------------------------------------------------------------------- .../phoenix/pherf/rules/RulesApplier.java | 70 ++++++++++++++------ .../apache/phoenix/pherf/util/PhoenixUtil.java | 23 +++---- .../apache/phoenix/pherf/RuleGeneratorTest.java | 51 ++++++++++++-- .../test/resources/datamodel/test_schema.sql | 1 + .../test/resources/scenario/test_scenario.xml | 14 +++- 5 files changed, 116 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java index 4801081..06a046d 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java @@ -19,6 +19,7 @@ package org.apache.phoenix.pherf.rules; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.random.RandomDataGenerator; import org.apache.phoenix.pherf.PherfConstants; import org.apache.commons.lang.RandomStringUtils; @@ -97,13 +98,6 @@ public class RulesApplier { Column columnRule = getColumnForRule(ruleList, phxMetaColumn); value = getDataValue(columnRule); - synchronized (value) { - // Add the prefix to the value if it exists. - if (columnRule.getPrefix() != null) { - value.setValue(columnRule.getPrefix() + value.getValue()); - } - } - } else { logger.warn("Attempted to apply rule to data, but could not find a rule to match type:" + phxMetaColumn.getType() @@ -122,46 +116,55 @@ public class RulesApplier { */ public DataValue getDataValue(Column column) throws Exception{ DataValue data = null; + String prefix = ""; int length = column.getLength(); int nullChance = column.getNullChance(); List<DataValue> dataValues = column.getDataValues(); - // Return an empty value if we we fall within the configured probability + // Return an empty value if we fall within the configured probability of null if ((nullChance != Integer.MIN_VALUE) && (isValueNull(nullChance))) { return new DataValue(column.getType(), ""); } + if (column.getPrefix() != null) { + prefix = column.getPrefix(); + } + + if (prefix.length() >= length) { + logger.warn("You are attempting to generate data with a prefix (" + prefix + ") " + + "That is longer than expected overall field length (" + length + "). " + + "This will certainly lead to unexpected data values."); + } + switch (column.getType()) { case VARCHAR: // Use the specified data values from configs if they exist if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = generateDataValue(dataValues); + data = pickDataValueFromList(dataValues); } else { Preconditions.checkArgument(length > 0, "length needs to be > 0"); if (column.getDataSequence() == DataSequence.SEQUENTIAL) { data = getSequentialDataValue(column); } else { - String varchar = RandomStringUtils.randomAlphanumeric(length); - data = new DataValue(column.getType(), varchar); + data = getRandomDataValue(column); } } break; case CHAR: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = generateDataValue(dataValues); + data = pickDataValueFromList(dataValues); } else { Preconditions.checkArgument(length > 0, "length needs to be > 0"); if (column.getDataSequence() == DataSequence.SEQUENTIAL) { data = getSequentialDataValue(column); } else { - String varchar = RandomStringUtils.randomAlphanumeric(length); - data = new DataValue(column.getType(), varchar); + data = getRandomDataValue(column); } } break; case DECIMAL: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = generateDataValue(dataValues); + data = pickDataValueFromList(dataValues); } else { int precision = column.getPrecision(); double minDbl = column.getMinValue(); @@ -181,7 +184,7 @@ public class RulesApplier { break; case INTEGER: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = generateDataValue(dataValues); + data = pickDataValueFromList(dataValues); } else { int minInt = column.getMinValue(); int maxInt = column.getMaxValue(); @@ -192,7 +195,7 @@ public class RulesApplier { break; case DATE: if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { - data = generateDataValue(dataValues); + data = pickDataValueFromList(dataValues); } else { int minYear = column.getMinValue(); int maxYear = column.getMaxValue(); @@ -205,7 +208,8 @@ public class RulesApplier { default: break; } - Preconditions.checkArgument(data != null, "Data value could not be generated for some reason. Please check configs"); + Preconditions.checkArgument(data != null, + "Data value could not be generated for some reason. Please check configs"); return data; } @@ -246,7 +250,7 @@ public class RulesApplier { return (rndNull.nextInt(100) < chance); } - private DataValue generateDataValue(List<DataValue> values) throws Exception{ + private DataValue pickDataValueFromList(List<DataValue> values) throws Exception{ DataValue generatedDataValue = null; int sum = 0, count = 0; @@ -265,7 +269,7 @@ public class RulesApplier { int rndIndex = rndVal.nextInt(values.size()); DataValue valueRule = values.get(rndIndex); - generatedDataValue = generateDataValue(valueRule); + generatedDataValue = pickDataValueFromList(valueRule); // While it's possible to get here if you have a bunch of really small distributions, // It's just really unlikely. This is just a safety just so we actually pick a value. @@ -278,7 +282,7 @@ public class RulesApplier { return generatedDataValue; } - private DataValue generateDataValue(final DataValue valueRule) throws Exception{ + private DataValue pickDataValueFromList(final DataValue valueRule) throws Exception{ DataValue retValue = new DataValue(valueRule); // Path taken when configuration specifies a specific value to be taken with the <value> tag @@ -339,6 +343,14 @@ public class RulesApplier { } } + public Column getRule(Column phxMetaColumn) { + // Assume the first rule map + Map<DataTypeMapping, List> ruleMap = modelList.get(0); + + List<Column> ruleList = ruleMap.get(phxMetaColumn.getType()); + return getColumnForRule(ruleList, phxMetaColumn); + } + private Column getColumnForRule(List<Column> ruleList, Column phxMetaColumn) { // Column pointer to head of list @@ -371,7 +383,21 @@ public class RulesApplier { long inc = COUNTER.getAndIncrement(); String strInc = String.valueOf(inc); String varchar = RandomStringUtils.randomAlphanumeric(column.getLength() - strInc.length()); - data = new DataValue(column.getType(), strInc + varchar); + varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc + varchar : + strInc + varchar; + + // Truncate string back down if it exceeds length + varchar = StringUtils.left(varchar,column.getLength()); + data = new DataValue(column.getType(), varchar); return data; } + + private DataValue getRandomDataValue(Column column) { + String varchar = RandomStringUtils.randomAlphanumeric(column.getLength()); + varchar = (column.getPrefix() != null) ? column.getPrefix() + varchar : varchar; + + // Truncate string back down if it exceeds length + varchar = StringUtils.left(varchar, column.getLength()); + return new DataValue(column.getType(), varchar); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java index 1f610f6..064c604 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java @@ -33,7 +33,6 @@ import java.util.Properties; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; -// TODO This class needs to be cleanup up a bit. I just wanted to get an initial placeholder in. public class PhoenixUtil { private static final Logger logger = LoggerFactory.getLogger(PhoenixUtil.class); private static String zookeeper; @@ -111,7 +110,9 @@ public class PhoenixUtil { result = preparedStatement.execute(); connection.commit(); } finally { - preparedStatement.close(); + if(preparedStatement != null) { + preparedStatement.close(); + } } return result; } @@ -125,9 +126,15 @@ public class PhoenixUtil { connection.commit(); } catch (SQLException e) { e.printStackTrace(); + if(preparedStatement != null) { + logger.error("Failed to apply schema. Statement (" + preparedStatement.toString() + ")", + e.getMessage()); + } } finally { try { - preparedStatement.close(); + if (preparedStatement != null) { + preparedStatement.close(); + } } catch (SQLException e) { e.printStackTrace(); } @@ -199,7 +206,7 @@ public class PhoenixUtil { public synchronized List<Column> getColumnsFromPhoenix(String schemaName, String tableName, Connection connection) throws SQLException { - List<Column> columnList = new ArrayList<Column>(); + List<Column> columnList = new ArrayList<>(); ResultSet resultSet = null; try { resultSet = getColumnsMetaData(schemaName, tableName, connection); @@ -293,14 +300,6 @@ public class PhoenixUtil { executeStatement("UPDATE STATISTICS " + tableName, scenario); } - public MonitorManager loadCustomMonitors(MonitorManager manager) throws Exception { - Properties - properties = - PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES, false); - - return manager; - } - /** * Get explain plan for a query * http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index 936eedb..37bfd47 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -18,17 +18,13 @@ package org.apache.phoenix.pherf; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.TreeSet; +import org.apache.commons.lang3.StringUtils; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.DataSequence; @@ -43,6 +39,8 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.junit.Test; +import static org.junit.Assert.*; + public class RuleGeneratorTest { private static final String matcherScenario = PherfConstants.SCENARIO_ROOT_PATTERN + ".xml"; @@ -179,7 +177,8 @@ public class RuleGeneratorTest { } } - assertTrue("Expected count in increments did not match expected", testSet.size() == (threadCount * increments)); + assertTrue("Expected count in increments did not match expected", + testSet.size() == (threadCount * increments)); } @Test @@ -205,6 +204,46 @@ public class RuleGeneratorTest { } } + @Test + public void testRuleOverrides() throws Exception { + XMLConfigParser parser = new XMLConfigParser(".*test_scenario.xml"); + WriteWorkload loader = new WriteWorkload(parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + Scenario scenario = parser.getScenarios().get(0); + + // We should be able to find the correct rule based only on Type and Name combination + // Test CHAR + Column simPhxCol = new Column(); + simPhxCol.setName("OTHER_ID"); + simPhxCol.setType(DataTypeMapping.CHAR); + + // Get the rule we expect to match + Column rule = rulesApplier.getRule(simPhxCol); + assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName()); + assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType()); + assertEquals("Rule contains incorrect length.", rule.getLength(), 8); + assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "z0Oxx00"); + + DataValue value = rulesApplier.getDataForRule(scenario, simPhxCol); + assertEquals("Value returned does not match rule.", value.getValue().length(), 8); + + // Test VARCHAR with RANDOM and prefix + simPhxCol.setName("OLDVAL_STRING"); + simPhxCol.setType(DataTypeMapping.VARCHAR); + + // Get the rule we expect to match + rule = rulesApplier.getRule(simPhxCol); + assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName()); + assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType()); + assertEquals("Rule contains incorrect length.", rule.getLength(), 10); + assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "MYPRFX"); + + value = rulesApplier.getDataForRule(scenario, simPhxCol); + assertEquals("Value returned does not match rule.", value.getValue().length(), 10); + assertTrue("Value returned start with prefix.", + StringUtils.startsWith(value.getValue(), rule.getPrefix())); + } + /** * Asserts that the value field is between the min/max value fields * http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/datamodel/test_schema.sql ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql index 4e6b9d4..2cf8e9b 100644 --- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql @@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE ( TENANT_ID CHAR(15) NOT NULL, PARENT_ID CHAR(15) NOT NULL, CREATED_DATE DATE NOT NULL, + OTHER_ID CHAR(15), FIELD VARCHAR, OLDVAL_STRING VARCHAR, NEWVAL_STRING VARCHAR, http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/scenario/test_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index f23e71a..def55ca 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -112,7 +112,7 @@ <column> <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere --> <type>VARCHAR</type> - <length>15</length> + <length>10</length> <userDefined>true</userDefined> <dataSequence>RANDOM</dataSequence> <name>OLDVAL_STRING</name> @@ -123,7 +123,7 @@ <type>VARCHAR</type> <length>15</length> <userDefined>true</userDefined> - <dataSequence>SEQUENTIAL</dataSequence> + <dataSequence>RANDOM</dataSequence> <name>NEWVAL_STRING</name> <prefix>TSTPRFX</prefix> </column> @@ -145,7 +145,15 @@ <value>LMN</value> </datavalue> </valuelist> - </column> + </column> + <column> + <type>CHAR</type> + <userDefined>true</userDefined> + <dataSequence>SEQUENTIAL</dataSequence> + <length>8</length> + <name>OTHER_ID</name> + <prefix>z0Oxx00</prefix> + </column> </datamapping> <scenarios> <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">
