This is an automated email from the ASF dual-hosted git repository. vjasani pushed a commit to branch 5.1 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 349040aedbefaf5e3941aa15a3816f438f90b226 Author: jpisaac <jacobpisaa...@gmail.com> AuthorDate: Wed Mar 31 21:01:02 2021 -0700 PHOENIX-6429 Add support for global connections and sequential data generators --- .../java/org/apache/phoenix/query/BaseTest.java | 2 +- .../org/apache/phoenix/pherf/ResultBaseTestIT.java | 5 +- .../MultiTenantOperationBaseIT.java | 11 +++ .../apache/phoenix/pherf/configuration/Ddl.java | 14 +++- .../phoenix/pherf/configuration/LoadProfile.java | 14 +++- .../phoenix/pherf/configuration/TenantGroup.java | 16 +++- .../apache/phoenix/pherf/rules/RulesApplier.java | 86 ++++++++++++---------- .../pherf/rules/SequentialDateDataGenerator.java | 68 +++++++++++++++++ .../pherf/rules/SequentialListDataGenerator.java | 66 +++++++++++++++++ .../rules/SequentialVarcharDataGenerator.java | 75 +++++++++++++++++++ .../tenantoperation/IdleTimeOperationSupplier.java | 3 +- .../PreScenarioOperationSupplier.java | 33 +++++---- .../mt/tenantoperation/QueryOperationSupplier.java | 16 +++- .../TenantOperationEventGenerator.java | 4 +- .../mt/tenantoperation/TenantOperationFactory.java | 5 +- .../tenantoperation/UpsertOperationSupplier.java | 16 +++- .../UserDefinedOperationSupplier.java | 2 + .../phoenix/pherf/ConfigurationParserTest.java | 27 ++++++- .../apache/phoenix/pherf/RuleGeneratorTest.java | 5 +- .../rules/SequentialDateDataGeneratorTest.java | 78 ++++++++++++++++++++ .../rules/SequentialListDataGeneratorTest.java | 86 ++++++++++++++++++++++ .../rules/SequentialVarcharDataGeneratorTest.java | 68 +++++++++++++++++ .../src/test/resources/datamodel/test_schema.sql | 3 +- .../scenario/malicious_scenario_with_dtd.xml | 2 +- .../src/test/resources/scenario/test_scenario.xml | 21 +++++- .../scenario/test_workload_with_load_profile.xml | 81 +++++++++++++++----- 26 files changed, 703 insertions(+), 104 deletions(-) diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index d8caa57..18401aa 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -956,7 +956,7 @@ public abstract class BaseTest { String fullTableName = SchemaUtil.getEscapedTableName( rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM), rs.getString(PhoenixDatabaseMetaData.TABLE_NAME)); - String ddl = "DROP " + rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE) + " " + fullTableName; + String ddl = "DROP " + rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE) + " " + fullTableName + " CASCADE"; String tenantId = rs.getString(1); if (tenantId != null && !tenantId.equals(lastTenantId)) { if (lastTenantId != null) { diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java index 75d6f6b..f3d7cba 100644 --- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java @@ -30,7 +30,6 @@ import org.apache.phoenix.pherf.configuration.XMLConfigParser; import org.apache.phoenix.pherf.result.ResultUtil; import org.apache.phoenix.pherf.schema.SchemaReader; import org.apache.phoenix.pherf.util.PhoenixUtil; -import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.After; import org.junit.AfterClass; @@ -39,8 +38,8 @@ import org.junit.experimental.categories.Category; @Category(NeedsOwnMiniClusterTest.class) public class ResultBaseTestIT extends ParallelStatsDisabledIT { - protected static final String matcherScenario = ".*scenario/.*test_scenario.*xml"; - protected static final String matcherSchema = ".*datamodel/.*test_schema.*sql"; + protected static final String matcherScenario = ".*scenario/.*test_scenario.xml"; + protected static final String matcherSchema = ".*datamodel/.*test_schema.sql"; protected static PhoenixUtil util = PhoenixUtil.create(true); protected static Properties properties; diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java index bcdbdca..f200381 100644 --- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java @@ -19,6 +19,8 @@ package org.apache.phoenix.pherf.workload.mt.tenantoperation; +import org.apache.hadoop.hbase.HConstants; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.pherf.PherfConstants; import org.apache.phoenix.pherf.XMLConfigParserTest; @@ -26,7 +28,11 @@ import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.XMLConfigParser; import org.apache.phoenix.pherf.schema.SchemaReader; import org.apache.phoenix.pherf.util.PhoenixUtil; +import org.junit.After; +import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -37,6 +43,7 @@ import java.util.Properties; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@Category(NeedsOwnMiniClusterTest.class) public class MultiTenantOperationBaseIT extends ParallelStatsDisabledIT { enum TestOperationGroup { upsertOp, queryOp1, queryOp2, idleOp, udfOp @@ -76,4 +83,8 @@ public class MultiTenantOperationBaseIT extends ParallelStatsDisabledIT { return XMLConfigParser.readDataModel(p); } + @AfterClass public static synchronized void tearDown() throws Exception { + dropNonSystemTables(); + } + } diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java index 3af8c3f..b60508d 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java @@ -23,7 +23,8 @@ import javax.xml.bind.annotation.XmlAttribute; public class Ddl { private String statement; private String tableName; - + private boolean useGlobalConnection; + public Ddl() { } @@ -55,7 +56,16 @@ public class Ddl { public void setTableName(String tableName) { this.tableName = tableName; } - + + @XmlAttribute + public boolean isUseGlobalConnection() { + return useGlobalConnection; + } + + public void setUseGlobalConnection(boolean useGlobalConnection) { + this.useGlobalConnection = useGlobalConnection; + } + public String toString(){ if (statement.contains("?")) { return statement.replace("?", tableName); diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java index 3116244..c66bbee 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java @@ -24,8 +24,8 @@ import java.util.List; @XmlType public class LoadProfile { private static final int MIN_BATCH_SIZE = 1; - private static final String DEFAULT_TENANT_ID_FMT = "00D%s%07d"; - private static final int DEFAULT_GROUP_ID_LEN = 5; + private static final String DEFAULT_TENANT_ID_FMT = "T%s%08d"; + private static final int DEFAULT_GROUP_ID_LEN = 6; private static final int DEFAULT_TENANT_ID_LEN = 15; // Holds the batch size to be used in upserts. @@ -37,7 +37,15 @@ public class LoadProfile { * TenantId format should typically have 2 parts - * 1. string fmt - that hold the tenant group id. * 2. int fmt - that holds a random number between 1 and max tenants - * for e.g DEFAULT_TENANT_ID_FMT = "00D%s%07d"; + * for e.g DEFAULT_TENANT_ID_FMT = "T%s%08d"; + * + * When the Tenant Group is configured to use a global connection, + * for now this is modelled as a special tenant whose id will translate to "TGLOBAL00000001" + * since the group id => "GLOBAL" and num tenants = 1. + * For now this is a hack/temporary workaround. + * + * TODO : + * Ideally it needs to be built into the framework and injected during event generation. */ private String tenantIdFormat; private int groupIdLength; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java index 0656917..d066cd5 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java @@ -21,13 +21,15 @@ package org.apache.phoenix.pherf.configuration; import javax.xml.bind.annotation.XmlAttribute; public class TenantGroup { + public static final String DEFAULT_GLOBAL_ID = "GLOBAL"; private String id; private int weight; private int numTenants; + private boolean useGlobalConnection; @XmlAttribute public String getId() { - return id; + return useGlobalConnection ? DEFAULT_GLOBAL_ID: id; } public void setId(String id) { @@ -36,7 +38,7 @@ public class TenantGroup { @XmlAttribute public int getWeight() { - return weight; + return useGlobalConnection ? 100 : weight; } public void setWeight(int weight) { @@ -44,9 +46,17 @@ public class TenantGroup { } @XmlAttribute - public int getNumTenants() { return numTenants; } + public int getNumTenants() { return useGlobalConnection ? 1 : numTenants; } public void setNumTenants(int numTenants) { this.numTenants = numTenants; } + @XmlAttribute + public boolean isUseGlobalConnection() { + return useGlobalConnection; + } + + public void setUseGlobalConnection(boolean useGlobalConnection) { + this.useGlobalConnection = useGlobalConnection; + } } diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java index e7a70ac..15d40c3 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java @@ -35,12 +35,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; public class RulesApplier { private static final Logger LOGGER = LoggerFactory.getLogger(RulesApplier.class); - private static final AtomicLong COUNTER = new AtomicLong(0); // Used to bail out of random distribution if it takes too long // This should never happen when distributions add up to 100 @@ -57,7 +56,7 @@ public class RulesApplier { private String cachedScenarioOverrideName; private Map<DataTypeMapping, List> scenarioOverrideMap; - private Map<Column,RuleBasedDataGenerator> columnRuleBasedDataGeneratorMap = new HashMap<>(); + private ConcurrentHashMap<String,RuleBasedDataGenerator> columnRuleBasedDataGeneratorMap = new ConcurrentHashMap<>(); // Since rules are only relevant for a given data model, // added a constructor to support a single data model => RulesApplier(DataModel model) @@ -163,7 +162,6 @@ public class RulesApplier { // Assume the first rule map Map<DataTypeMapping, List> ruleMap = modelList.get(0); List<Column> ruleList = ruleMap.get(phxMetaColumn.getType()); - //LOGGER.info(String.format("Did not found a correct override column rule, %s, %s", phxMetaColumn.getName(), phxMetaColumn.getType())); // Make sure Column from Phoenix Metadata matches a rule column if (ruleList != null && ruleList.contains(phxMetaColumn)) { @@ -173,9 +171,9 @@ public class RulesApplier { value = getDataValue(columnRule); } else { - LOGGER.warn("Attempted to apply rule to data, but could not find a rule to match type:" - + phxMetaColumn.getType() - ); + LOGGER.warn(String.format("Attempted to apply rule to data, " + + "but could not find a rule to match type %s on %s", + phxMetaColumn.getType(), phxMetaColumn.getName())); } } @@ -189,7 +187,7 @@ public class RulesApplier { * @param column {@link org.apache.phoenix.pherf.configuration.Column} Column rule to get data for * @return {@link org.apache.phoenix.pherf.rules.DataValue} {@code Container Type --> Value mapping } */ - public DataValue getDataValue(Column column) throws Exception{ + public DataValue getDataValue(Column column) throws Exception { DataValue data = null; String prefix = ""; int length = column.getLength(); @@ -216,15 +214,14 @@ public class RulesApplier { case VARBINARY: case CHAR: // Use the specified data values from configs if they exist - if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { + if (DataSequence.SEQUENTIAL.equals(column.getDataSequence())) { + RuleBasedDataGenerator generator = getRuleBasedDataGeneratorForColumn(column); + data = generator.getDataValue(); + } else if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { data = pickDataValueFromList(dataValues); } else { Preconditions.checkArgument(length > 0, "length needs to be > 0"); - if (column.getDataSequence() == DataSequence.SEQUENTIAL) { - data = getSequentialVarcharDataValue(column); - } else { - data = getRandomDataValue(column); - } + data = getRandomDataValue(column); } break; case VARCHAR_ARRAY: @@ -295,6 +292,9 @@ public class RulesApplier { data = pickDataValueFromList(dataValues); // Check if date has right format or not data.setValue(checkDatePattern(data.getValue())); + } else if(DataSequence.SEQUENTIAL.equals(column.getDataSequence())) { + RuleBasedDataGenerator generator = getRuleBasedDataGeneratorForColumn(column); + data = generator.getDataValue(); } else if (column.getUseCurrentDate() != true){ int minYear = (int) column.getMinValue(); int maxYear = (int) column.getMaxValue(); @@ -535,28 +535,6 @@ public class RulesApplier { return ruleAppliedColumn; } - /** - * Add a numerically increasing counter onto the and of a random string. - * Incremented counter should be thread safe. - * - * @param column {@link org.apache.phoenix.pherf.configuration.Column} - * @return {@link org.apache.phoenix.pherf.rules.DataValue} - */ - private DataValue getSequentialVarcharDataValue(Column column) { - DataValue data = null; - long inc = COUNTER.getAndIncrement(); - String strInc = String.valueOf(inc); - int paddedLength = column.getLengthExcludingPrefix(); - String strInc1 = StringUtils.leftPad(strInc, paddedLength, "0"); - String strInc2 = StringUtils.right(strInc1, column.getLengthExcludingPrefix()); - String varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc2: - strInc2; - - // Truncate string back down if it exceeds length - varchar = StringUtils.left(varchar,column.getLength()); - data = new DataValue(column.getType(), varchar); - return data; - } private DataValue getRandomDataValue(Column column) { String varchar = RandomStringUtils.randomAlphanumeric(column.getLength()); @@ -568,11 +546,39 @@ public class RulesApplier { } private RuleBasedDataGenerator getRuleBasedDataGeneratorForColumn(Column column) { - RuleBasedDataGenerator generator = columnRuleBasedDataGeneratorMap.get(column); + RuleBasedDataGenerator generator = columnRuleBasedDataGeneratorMap.get(column.getName()); if(generator == null) { - //For now we only have one of these, likely this should replace all all the methods - generator = new SequentialIntegerDataGenerator(column); - columnRuleBasedDataGeneratorMap.put(column,generator); + //For now we only have couple of these, likely this should replace for all the methods + switch (column.getType()) { + case VARCHAR: + case VARBINARY: + case CHAR: + if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) { + generator = new SequentialListDataGenerator(column); + } else { + generator = new SequentialVarcharDataGenerator(column); + } + break; + case DATE: + case TIMESTAMP: + generator = new SequentialDateDataGenerator(column); + break; + case BIGINT: + case INTEGER: + case TINYINT: + case UNSIGNED_LONG: + generator = new SequentialIntegerDataGenerator(column); + break; + default: + throw new IllegalArgumentException( + String.format("No rule based generator supported for column type %s on %s", + column.getType(), column.getName())); + } + RuleBasedDataGenerator oldGenerator = columnRuleBasedDataGeneratorMap.putIfAbsent(column.getName(),generator); + if (oldGenerator != null) { + // Another thread succeeded in registering their generator first, so let's use that. + generator = oldGenerator; + } } return generator; } diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java new file mode 100644 index 0000000..63328d0 --- /dev/null +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.apache.phoenix.pherf.configuration.DataTypeMapping; + +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A generator for sequentially increasing dates. + * For now the increments are fixed at 1 second. + */ +public class SequentialDateDataGenerator implements RuleBasedDataGenerator { + private static DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private final Column columnRule; + private final AtomicInteger counter; + private final LocalDateTime startDateTime = new LocalDateTime(); + + public SequentialDateDataGenerator(Column columnRule) { + Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL); + Preconditions.checkArgument(isDateType(columnRule.getType())); + this.columnRule = columnRule; + counter = new AtomicInteger(0); + } + + /** + * Note that this method rolls over for attempts to get larger than maxValue + * @return new DataValue + */ + @Override + public DataValue getDataValue() { + LocalDateTime newDateTime = startDateTime.plusSeconds(counter.getAndIncrement()); + String formattedDateTime = newDateTime.toString(FMT); + return new DataValue(columnRule.getType(), formattedDateTime); + } + + boolean isDateType(DataTypeMapping mapping) { + switch (mapping) { + case DATE: + case TIMESTAMP: + return true; + default: + return false; + } + } +} diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java new file mode 100644 index 0000000..ddcc354 --- /dev/null +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.apache.phoenix.pherf.configuration.DataTypeMapping; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * A generator to round robin thru a list of values. + */ + +public class SequentialListDataGenerator implements RuleBasedDataGenerator { + private final Column columnRule; + private final AtomicLong counter; + + public SequentialListDataGenerator(Column columnRule) { + Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL); + Preconditions.checkArgument(columnRule.getDataValues().size() > 0); + Preconditions.checkArgument(isAllowedType(columnRule.getType())); + this.columnRule = columnRule; + counter = new AtomicLong(0); + } + + /** + * Note that this method rolls over for attempts to get larger than maxValue + * @return new DataValue + */ + @Override + public DataValue getDataValue() { + long pos = counter.getAndIncrement(); + int index = (int) pos % columnRule.getDataValues().size(); + return columnRule.getDataValues().get(index); + } + + boolean isAllowedType(DataTypeMapping mapping) { + // For now only varchar list are supported + switch (mapping) { + case VARCHAR: + case VARBINARY: + case CHAR: + return true; + default: + return false; + } + } +} diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java new file mode 100644 index 0000000..9ae23b8 --- /dev/null +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.apache.phoenix.pherf.configuration.DataTypeMapping; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A generator for sequentially increasing varchar values. + */ +public class SequentialVarcharDataGenerator implements RuleBasedDataGenerator { + private final Column columnRule; + private final AtomicLong counter; + + public SequentialVarcharDataGenerator(Column columnRule) { + Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL); + Preconditions.checkArgument(isVarcharType(columnRule.getType())); + this.columnRule = columnRule; + counter = new AtomicLong(0); + } + + /** + * Add a numerically increasing counter onto the and of a random string. + * Incremented counter should be thread safe. + * + * @return {@link org.apache.phoenix.pherf.rules.DataValue} + */ + @Override + public DataValue getDataValue() { + DataValue data = null; + long inc = counter.getAndIncrement(); + String strInc = String.valueOf(inc); + int paddedLength = columnRule.getLengthExcludingPrefix(); + String strInc1 = StringUtils.leftPad(strInc, paddedLength, "x"); + String strInc2 = StringUtils.right(strInc1, columnRule.getLengthExcludingPrefix()); + String varchar = (columnRule.getPrefix() != null) ? columnRule.getPrefix() + strInc2: + strInc2; + + // Truncate string back down if it exceeds length + varchar = StringUtils.left(varchar,columnRule.getLength()); + data = new DataValue(columnRule.getType(), varchar); + return data; + } + + boolean isVarcharType(DataTypeMapping mapping) { + switch (mapping) { + case VARCHAR: + case VARBINARY: + case CHAR: + return true; + default: + return false; + } + } +} diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java index ab45c27..c65f430 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java @@ -25,6 +25,7 @@ import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.pherf.workload.mt.IdleTimeOperation; import org.apache.phoenix.pherf.workload.mt.OperationStats; +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,7 @@ class IdleTimeOperationSupplier extends BaseOperationSupplier { @Override public OperationStats apply(final TenantOperationInfo input) { - + Preconditions.checkNotNull(input); final IdleTimeOperation operation = (IdleTimeOperation) input.getOperation(); final IdleTime idleTime = operation.getIdleTime(); diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java index 4f1e3e3..a323a36 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java @@ -18,6 +18,7 @@ package org.apache.phoenix.pherf.workload.mt.tenantoperation; +import org.apache.phoenix.pherf.configuration.TenantGroup; import org.apache.phoenix.thirdparty.com.google.common.base.Function; import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.Ddl; @@ -25,6 +26,7 @@ import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.pherf.workload.mt.OperationStats; import org.apache.phoenix.pherf.workload.mt.PreScenarioOperation; +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,32 +50,37 @@ class PreScenarioOperationSupplier extends BaseOperationSupplier { @Override public OperationStats apply(final TenantOperationInfo input) { + Preconditions.checkNotNull(input); final PreScenarioOperation operation = (PreScenarioOperation) input.getOperation(); - final String tenantId = input.getTenantId(); final String tenantGroup = input.getTenantGroupId(); final String opGroup = input.getOperationGroupId(); final String tableName = input.getTableName(); final String scenarioName = input.getScenarioName(); - final String opName = String.format("%s:%s:%s:%s:%s", - scenarioName, tableName, opGroup, tenantGroup, tenantId); + final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0); long startTime = EnvironmentEdgeManager.currentTimeMillis(); int status = 0; if (!operation.getPreScenarioDdls().isEmpty()) { - try (Connection conn = phoenixUtil.getConnection(tenantId)) { - for (Ddl ddl : operation.getPreScenarioDdls()) { - LOGGER.info("\nExecuting DDL:" + ddl + " on tenantId:" + tenantId); - phoenixUtil.executeStatement(ddl.toString(), conn); + for (Ddl ddl : operation.getPreScenarioDdls()) { + // TODO: + // Ideally the fact that the op needs to executed using global connection + // needs to be built into the framework and injected during event generation. + // For now a special tenant whose id = "TGLOBAL00000001" will be logged. + final String tenantId = isTenantGroupGlobal || ddl.isUseGlobalConnection() ? null : input.getTenantId(); + final String opName = String.format("%s:%s:%s:%s:%s", + scenarioName, tableName, opGroup, tenantGroup, input.getTenantId()); + + try (Connection conn = phoenixUtil.getConnection(tenantId)) { + LOGGER.info("\nExecuting DDL:" + ddl + ", OPERATION:" + opName); + String sql = ddl.toString(); + phoenixUtil.executeStatement(sql, conn); if (ddl.getStatement().toUpperCase().contains(phoenixUtil.ASYNC_KEYWORD)) { phoenixUtil.waitForAsyncIndexToFinish(ddl.getTableName()); } + } catch (Exception e) { + LOGGER.error("Operation " + opName + " failed with exception ", e); + status = -1; } - } catch (SQLException sqle) { - LOGGER.error("Operation " + opName + " failed with exception ", sqle); - status = -1; - } catch (Exception e) { - LOGGER.error("Operation " + opName + " failed with exception ", e); - status = -1; } } long totalDuration = EnvironmentEdgeManager.currentTimeMillis() - startTime; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java index 88eec68..99188ce 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java @@ -18,6 +18,7 @@ package org.apache.phoenix.pherf.workload.mt.tenantoperation; +import org.apache.phoenix.pherf.configuration.TenantGroup; import org.apache.phoenix.thirdparty.com.google.common.base.Function; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.pherf.configuration.DataModel; @@ -26,6 +27,7 @@ import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.pherf.workload.mt.OperationStats; import org.apache.phoenix.pherf.workload.mt.QueryOperation; +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,17 +52,23 @@ public class QueryOperationSupplier extends BaseOperationSupplier { @Override public OperationStats apply(final TenantOperationInfo input) { - + Preconditions.checkNotNull(input); final QueryOperation operation = (QueryOperation) input.getOperation(); + final Query query = operation.getQuery(); final String tenantGroup = input.getTenantGroupId(); final String opGroup = input.getOperationGroupId(); - final String tenantId = input.getTenantId(); final String scenarioName = input.getScenarioName(); final String tableName = input.getTableName(); - final Query query = operation.getQuery(); + + // TODO: + // Ideally the fact that the op needs to executed using global connection + // needs to be built into the framework and injected during event generation. + // For now a special tenant whose id = "TGLOBAL00000001" will be logged. + final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0); + final String tenantId = isTenantGroupGlobal || query.isUseGlobalConnection() ? null : input.getTenantId(); String opName = String.format("%s:%s:%s:%s:%s", scenarioName, tableName, - opGroup, tenantGroup, tenantId); + opGroup, tenantGroup, input.getTenantId()); LOGGER.info("\nExecuting query " + query.getStatement()); long startTime = 0; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java index 676c510..de8f95a 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java @@ -120,10 +120,10 @@ public class TenantOperationEventGenerator Operation op = operationMap.get(opId); int numTenants = tenantGroupMap.get(tenantGroupId).getNumTenants(); - String tenantIdPrefix = Strings.padStart(tenantGroupId, loadProfile.getGroupIdLength(), '0'); + String tenantIdPrefix = Strings.padStart(tenantGroupId, loadProfile.getGroupIdLength(), 'x'); String formattedTenantId = String.format(loadProfile.getTenantIdFormat(), tenantIdPrefix.substring(0, loadProfile.getGroupIdLength()), RANDOM.nextInt(numTenants)); - String paddedTenantId = Strings.padStart(formattedTenantId, loadProfile.getTenantIdLength(), '0'); + String paddedTenantId = Strings.padStart(formattedTenantId, loadProfile.getTenantIdLength(), 'x'); String tenantId = paddedTenantId.substring(0, loadProfile.getTenantIdLength()); TenantOperationInfo sample = new TenantOperationInfo(modelName, scenarioName, tableName, diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java index 365984f..6e86e14 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java @@ -43,6 +43,7 @@ import org.apache.phoenix.pherf.workload.mt.EventGenerator; import org.apache.phoenix.pherf.workload.mt.IdleTimeOperation; import org.apache.phoenix.pherf.workload.mt.Operation; import org.apache.phoenix.pherf.workload.mt.OperationStats; +import org.apache.phoenix.pherf.workload.mt.PreScenarioOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -274,8 +275,8 @@ public class TenantOperationFactory { // Check if the scenario has a PRE_RUN operation. if (preRunOpSupplier != null) { // Initialize the tenant using the pre scenario ddls. - final org.apache.phoenix.pherf.workload.mt.PreScenarioOperation - operation = new org.apache.phoenix.pherf.workload.mt.PreScenarioOperation() { + final PreScenarioOperation + operation = new PreScenarioOperation() { @Override public List<Ddl> getPreScenarioDdls() { List<Ddl> ddls = scenario.getPreScenarioDdls(); return ddls == null ? Lists.<Ddl>newArrayList() : ddls; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java index 5b25c12..30f953b 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java @@ -18,6 +18,7 @@ package org.apache.phoenix.pherf.workload.mt.tenantoperation; +import org.apache.phoenix.pherf.configuration.TenantGroup; import org.apache.phoenix.thirdparty.com.google.common.base.Function; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataModel; @@ -26,6 +27,7 @@ import org.apache.phoenix.pherf.configuration.Upsert; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.pherf.workload.mt.OperationStats; import org.apache.phoenix.pherf.workload.mt.UpsertOperation; +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,22 +54,28 @@ class UpsertOperationSupplier extends BaseOperationSupplier { @Override public OperationStats apply(final TenantOperationInfo input) { - + Preconditions.checkNotNull(input); final int batchSize = loadProfile.getBatchSize(); final boolean useBatchApi = batchSize != 0; final int rowCount = useBatchApi ? batchSize : 1; final UpsertOperation operation = (UpsertOperation) input.getOperation(); + final Upsert upsert = operation.getUpsert(); final String tenantGroup = input.getTenantGroupId(); final String opGroup = input.getOperationGroupId(); - final String tenantId = input.getTenantId(); - final Upsert upsert = operation.getUpsert(); final String tableName = input.getTableName(); final String scenarioName = input.getScenarioName(); final List<Column> columns = upsert.getColumn(); + // TODO: + // Ideally the fact that the op needs to executed using global connection + // needs to be built into the framework and injected during event generation. + // For now a special tenant whose id = "TGLOBAL00000001" will be logged. + + final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0); + final String tenantId = isTenantGroupGlobal || upsert.isUseGlobalConnection() ? null : input.getTenantId(); final String opName = String.format("%s:%s:%s:%s:%s", - scenarioName, tableName, opGroup, tenantGroup, tenantId); + scenarioName, tableName, opGroup, tenantGroup, input.getTenantId()); long rowsCreated = 0; long startTime = 0, duration, totalDuration; diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java index ae8ce6f..f1f650a 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java @@ -24,6 +24,7 @@ import org.apache.phoenix.pherf.configuration.Scenario; import org.apache.phoenix.pherf.util.PhoenixUtil; import org.apache.phoenix.pherf.workload.mt.OperationStats; import org.apache.phoenix.pherf.workload.mt.UserDefinedOperation; +import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions; import org.apache.phoenix.util.EnvironmentEdgeManager; /** @@ -40,6 +41,7 @@ class UserDefinedOperationSupplier extends BaseOperationSupplier { return new Function<TenantOperationInfo, OperationStats>() { @Override public OperationStats apply(final TenantOperationInfo input) { + Preconditions.checkNotNull(input); // TODO : implement user defined operation invocation. long startTime = EnvironmentEdgeManager.currentTimeMillis(); long duration = EnvironmentEdgeManager.currentTimeMillis() - startTime; diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java index 929b7fa..78f50a6 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Set; +import org.apache.phoenix.pherf.rules.RulesApplier; import org.apache.phoenix.thirdparty.com.google.common.collect.Sets; import org.apache.phoenix.pherf.configuration.*; import org.apache.phoenix.pherf.rules.DataValue; @@ -162,6 +163,30 @@ public class ConfigurationParserTest extends ResultBaseTest { 1, testScenarioWithLoadProfile.getQuerySet().size()); assertEquals("#Queries within the first querySet is not as expected ", 2, testScenarioWithLoadProfile.getQuerySet().get(0).getQuery().size()); + + // Test configuration for global connection + Scenario testScenarioWithGlobalConn = scenarioList.get(2); + LoadProfile loadProfileWithGlobalConn = testScenarioWithGlobalConn.getLoadProfile(); + assertEquals("batch size not as expected: ", + 1, loadProfileWithGlobalConn.getBatchSize()); + assertEquals("num operations not as expected: ", + 1000, loadProfileWithGlobalConn.getNumOperations()); + assertEquals("tenant group size is not as expected: ", + 1, loadProfileWithGlobalConn.getTenantDistribution().size()); + assertEquals("global tenant is not as expected: ", + 1, loadProfileWithGlobalConn.getTenantDistribution().get(0).getNumTenants()); + assertEquals("global tenant id is not as expected: ", + "GLOBAL", loadProfileWithGlobalConn.getTenantDistribution().get(0).getId()); + assertEquals("global tenant weight is not as expected: ", + 100, loadProfileWithGlobalConn.getTenantDistribution().get(0).getWeight()); + assertEquals("operation group size is not as expected: ", + 1,loadProfileWithGlobalConn.getOpDistribution().size()); + assertEquals("UpsertSet size is not as expected ", + 1, testScenarioWithGlobalConn.getUpserts().size()); + assertEquals("#Column within the first upsert is not as expected ", + 7, testScenarioWithGlobalConn.getUpserts().get(0).getColumn().size()); + assertEquals("Upsert operation not using global connection as expected ", + true, testScenarioWithGlobalConn.getUpserts().get(0).isUseGlobalConnection()); } private URL getResourceUrl(String resourceName) { @@ -186,7 +211,7 @@ public class ConfigurationParserTest extends ResultBaseTest { private void assertDateValue(List<Column> dataMappingColumns) { for (Column dataMapping : dataMappingColumns) { if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName() - .equals("CREATED_DATE"))) { + .equals("SOME_DATE"))) { // First rule should have min/max set assertNotNull(dataMapping.getDataValues().get(0).getMinValue()); assertNotNull(dataMapping.getDataValues().get(0).getMaxValue()); diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index c439d38..c62922b 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -59,7 +59,7 @@ public class RuleGeneratorTest { RulesApplier rulesApplier = loader.getRulesApplier(); for (Column dataMapping : model.getDataMappingColumns()) { - if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName().equals("CREATED_DATE"))) { + if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName().equals("SOME_DATE"))) { // Test directly through generator method and that it converts to Phoenix type assertRandomDateValue(dataMapping, rulesApplier); @@ -92,7 +92,8 @@ public class RuleGeneratorTest { for (Column dataMapping : model.getDataMappingColumns()) { if ((dataMapping.getType() == DataTypeMapping.DATE) - && (dataMapping.getUseCurrentDate() == true)) { + && (dataMapping.getUseCurrentDate() == true) + && (dataMapping.getDataSequence() != DataSequence.SEQUENTIAL)) { // Generate the date using rules DataValue value = rulesApplier.getDataValue(dataMapping); diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java new file mode 100644 index 0000000..3cc75e4 --- /dev/null +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.junit.Test; + +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE; +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class SequentialDateDataGeneratorTest { + SequentialDateDataGenerator generator; + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonSequential() { + Column columnA = new Column(); + columnA.setType(DATE); + columnA.setDataSequence(DataSequence.RANDOM); + + //should reject this Column + generator = new SequentialDateDataGenerator(columnA); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonDate() { + Column columnA = new Column(); + columnA.setType(VARCHAR); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + + //should reject this Column + generator = new SequentialDateDataGenerator(columnA); + } + + @Test + public void testGetDataValue() { + DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + Column columnA = new Column(); + columnA.setType(DATE); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + LocalDateTime startDateTime = new LocalDateTime(); + + // The increments are the of 1 sec units + generator = new SequentialDateDataGenerator(columnA); + DataValue result1 = generator.getDataValue(); + LocalDateTime result1LocalTime = FMT.parseDateTime(result1.getValue()).toLocalDateTime(); + assertFalse(result1LocalTime.isBefore(startDateTime)); + DataValue result2 = generator.getDataValue(); + LocalDateTime result2LocalTime = FMT.parseDateTime(result2.getValue()).toLocalDateTime(); + assertEquals(result2LocalTime.minusSeconds(1), result1LocalTime); + DataValue result3 = generator.getDataValue(); + LocalDateTime result3LocalTime = FMT.parseDateTime(result3.getValue()).toLocalDateTime(); + assertEquals(result3LocalTime.minusSeconds(1), result2LocalTime); + DataValue result4 = generator.getDataValue(); + LocalDateTime result4LocalTime = FMT.parseDateTime(result4.getValue()).toLocalDateTime(); + assertEquals(result4LocalTime.minusSeconds(1), result3LocalTime); + } +} diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java new file mode 100644 index 0000000..d590167 --- /dev/null +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE; +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR; +import static org.junit.Assert.assertEquals; + +public class SequentialListDataGeneratorTest { + SequentialListDataGenerator generator; + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonSequential() { + Column columnA = new Column(); + columnA.setType(VARCHAR); + columnA.setDataSequence(DataSequence.RANDOM); + + //should reject this Column + generator = new SequentialListDataGenerator(columnA); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonVarchar() { + DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + LocalDateTime startDateTime = new LocalDateTime(); + String formattedDateTime = startDateTime.toString(FMT); + Column columnA = new Column(); + columnA.setType(DATE); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + List<DataValue> values = new ArrayList<>(); + values.add(new DataValue(DATE, formattedDateTime)); + values.add(new DataValue(DATE, formattedDateTime)); + values.add(new DataValue(DATE, formattedDateTime)); + columnA.setDataValues(values); + + //should reject this Column + generator = new SequentialListDataGenerator(columnA); + } + + @Test + public void testGetDataValue() { + Column columnA = new Column(); + columnA.setType(VARCHAR); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + List<DataValue> values = new ArrayList<>(); + values.add(new DataValue(VARCHAR, "A")); + values.add(new DataValue(VARCHAR, "B")); + values.add(new DataValue(VARCHAR, "C")); + columnA.setDataValues(values); + + generator = new SequentialListDataGenerator(columnA); + DataValue result1 = generator.getDataValue(); + assertEquals("A", result1.getValue()); + DataValue result2 = generator.getDataValue(); + assertEquals("B", result2.getValue()); + DataValue result3 = generator.getDataValue(); + assertEquals("C", result3.getValue()); + DataValue result4 = generator.getDataValue(); + assertEquals("A", result4.getValue()); + } +} diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java new file mode 100644 index 0000000..0157721 --- /dev/null +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.pherf.rules; + +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataSequence; +import org.junit.Test; + +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.INTEGER; +import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR; +import static org.junit.Assert.assertEquals; + +public class SequentialVarcharDataGeneratorTest { + SequentialVarcharDataGenerator generator; + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonSequential() { + Column columnA = new Column(); + columnA.setType(VARCHAR); + columnA.setDataSequence(DataSequence.RANDOM); + + //should reject this Column + generator = new SequentialVarcharDataGenerator(columnA); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectsNonVarchar() { + Column columnA = new Column(); + columnA.setType(INTEGER); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + + //should reject this Column + generator = new SequentialVarcharDataGenerator(columnA); + } + + @Test + public void testGetDataValue() { + Column columnA = new Column(); + columnA.setType(VARCHAR); + columnA.setLength(15); + columnA.setDataSequence(DataSequence.SEQUENTIAL); + + generator = new SequentialVarcharDataGenerator(columnA); + DataValue result1 = generator.getDataValue(); + assertEquals("xxxxxxxxxxxxxx0", result1.getValue()); + DataValue result2 = generator.getDataValue(); + assertEquals("xxxxxxxxxxxxxx1", result2.getValue()); + DataValue result3 = generator.getDataValue(); + assertEquals("xxxxxxxxxxxxxx2", result3.getValue()); + DataValue result4 = generator.getDataValue(); + assertEquals("xxxxxxxxxxxxxx3", result4.getValue()); + } +} diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql index a5a7274..9a1c856 100644 --- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql +++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql @@ -19,7 +19,8 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE ( TENANT_ID CHAR(15) NOT NULL, PARENT_ID CHAR(15) NOT NULL, CREATED_DATE DATE NOT NULL, - NOW_DATE DATE, + SOME_DATE DATE, + RND_DATE DATE, TS_DATE TIMESTAMP, PRESENT_DATE DATE, OTHER_ID CHAR(15), diff --git a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml index e8528f3..cc45395 100644 --- a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml +++ b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml @@ -40,7 +40,7 @@ least once. --> <scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno" - ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" + ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" rowCount="100" name="testMTDdlWriteScenario"> </scenario> diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index 853b857..db3caac 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -59,13 +59,14 @@ <column> <type>DATE</type> <!--SEQUENTIAL is unsupported for DATE --> + <userDefined>true</userDefined> <dataSequence>RANDOM</dataSequence> <!-- Number [0-100] that represents the probability of creating a null value --> <!-- The higher the number, the more like the value will returned will be null --> <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null --> <nullChance>0</nullChance> <useCurrentDate>true</useCurrentDate> - <name>NOW_DATE</name> + <name>RND_DATE</name> </column> <column> <type>DECIMAL</type> @@ -94,7 +95,20 @@ </column> <column> <type>DATE</type> + <userDefined>true</userDefined> + <!--SEQUENTIAL is now supported for DATE --> + <dataSequence>SEQUENTIAL</dataSequence> + <!-- Number [0-100] that represents the probability of creating a null value --> + <!-- The higher the number, the more like the value will returned will be null --> + <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null --> + <nullChance>0</nullChance> + <useCurrentDate>true</useCurrentDate> <name>CREATED_DATE</name> + </column> + <column> + <type>DATE</type> + <userDefined>true</userDefined> + <name>SOME_DATE</name> <minValue>1975</minValue> <maxValue>2025</maxValue> <valuelist> @@ -115,6 +129,7 @@ </column> <column> <type>DATE</type> + <userDefined>true</userDefined> <name>PRESENT_DATE</name> <minValue>1975</minValue> <maxValue>2025</maxValue> @@ -331,7 +346,7 @@ <scenario tableName="PHERF.TEST_VIEW" tenantId="xyzdefghijklmno" rowCount="100" name="testMTWriteScenario"> <preScenarioDdls> - <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" /> + <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" /> </preScenarioDdls> </scenario> <!-- Scenario level DDL that is dynamically executed before the Write Workload is run. @@ -344,7 +359,7 @@ <scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno" rowCount="100" name="testMTDdlWriteScenario"> <preScenarioDdls> - <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" /> + <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" /> </preScenarioDdls> </scenario> diff --git a/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml index 855c1fe..3a1c710 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml @@ -234,14 +234,14 @@ <loadProfile> <batchSize>1</batchSize> <numOperations>1000</numOperations> - <tenantDistribution id="t111" weight="10" numTenants="10"></tenantDistribution> - <tenantDistribution id="t112" weight="10" numTenants="10"></tenantDistribution> - <tenantDistribution id="t113" weight="80" numTenants="1"></tenantDistribution> - <opDistribution id="op111" weight="50"></opDistribution> - <opDistribution id="op112" weight="0"></opDistribution> - <opDistribution id="op113" weight="0"></opDistribution> - <opDistribution id="op114" weight="50"></opDistribution> - <opDistribution id="op115" weight="0"></opDistribution> + <tenantDistribution id="t111" weight="10" numTenants="10"/> + <tenantDistribution id="t112" weight="10" numTenants="10"/> + <tenantDistribution id="t113" weight="80" numTenants="1"/> + <opDistribution id="op111" weight="50"/> + <opDistribution id="op112" weight="0"/> + <opDistribution id="op113" weight="0"/> + <opDistribution id="op114" weight="50"/> + <opDistribution id="op115" weight="0"/> </loadProfile> @@ -283,7 +283,7 @@ </upserts> <idleTimes> - <idleTime id="op114" idleTime="50"></idleTime> + <idleTime id="op114" idleTime="50"/> </idleTimes> <udfs> <udf id="op115" > @@ -302,14 +302,14 @@ <loadProfile> <batchSize>5</batchSize> <numOperations>1000</numOperations> - <tenantDistribution id="t121" weight="10" numTenants="5"></tenantDistribution> - <tenantDistribution id="t122" weight="10" numTenants="5"></tenantDistribution> - <tenantDistribution id="t123" weight="80" numTenants="5"></tenantDistribution> - <opDistribution id="op121" weight="50"></opDistribution> - <opDistribution id="op122" weight="5"></opDistribution> - <opDistribution id="op123" weight="5"></opDistribution> - <opDistribution id="op124" weight="40"></opDistribution> - <opDistribution id="op125" weight="0"></opDistribution> + <tenantDistribution id="t121" weight="10" numTenants="5"/> + <tenantDistribution id="t122" weight="10" numTenants="5"/> + <tenantDistribution id="t123" weight="80" numTenants="5"/> + <opDistribution id="op121" weight="50"/> + <opDistribution id="op122" weight="5"/> + <opDistribution id="op123" weight="5"/> + <opDistribution id="op124" weight="40"/> + <opDistribution id="op125" weight="0"/> </loadProfile> <preScenarioDdls> @@ -350,7 +350,7 @@ </upserts> <idleTimes> - <idleTime id="op124" idleTime="100"></idleTime> + <idleTime id="op124" idleTime="100"/> </idleTimes> <querySet> <query id="op122" statement="select count(*) from PHERF.Z12"/> @@ -365,6 +365,51 @@ </udfs> </scenario> + <scenario tableName="PHERF.Z13" name="scenario_13"> + <loadProfile> + <batchSize>1</batchSize> + <numOperations>1000</numOperations> + <tenantDistribution useGlobalConnection="true"/> + <opDistribution id="op131" weight="100"/> + </loadProfile> + + <preScenarioDdls> + <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.Z13 (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE WHERE PARENT_ID = 'aAAyYhnNbBs9kWk'" /> + </preScenarioDdls> + + <upserts> + <upsert id="op131" useGlobalConnection="true"> + <column> + <type>CHAR</type> + <name>PARENT_ID_SEQ</name> + </column> + <column> + <type>DATE</type> + <name>CREATED_DATE</name> + </column> + <column> + <type>VARCHAR</type> + <name>FIELD</name> + </column> + <column> + <type>VARCHAR</type> + <name>OTHER_ID</name> + </column> + <column> + <type>VARCHAR</type> + <name>OLDVAL_STRING</name> + </column> + <column> + <type>VARCHAR</type> + <name>NEWVAL_STRING</name> + </column> + <column> + <type>VARCHAR</type> + <name>FIELD1</name> + </column> + </upsert> + </upserts> + </scenario> </scenarios> </datamodel>