This is an automated email from the ASF dual-hosted git repository.
yanxinyi pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 3f8a74e PHOENIX-6429 Add support for global connections and
sequential data generators
3f8a74e is described below
commit 3f8a74e4c3c73e92ed400791dd9da03153aacc9f
Author: jpisaac <[email protected]>
AuthorDate: Wed Mar 31 21:01:20 2021 -0700
PHOENIX-6429 Add support for global connections and sequential data
generators
---
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
.../org/apache/phoenix/pherf/ResultBaseTestIT.java | 5 +-
.../MultiTenantOperationBaseIT.java | 11 +++
.../apache/phoenix/pherf/configuration/Ddl.java | 14 +++-
.../phoenix/pherf/configuration/LoadProfile.java | 14 +++-
.../phoenix/pherf/configuration/TenantGroup.java | 16 +++-
.../apache/phoenix/pherf/rules/RulesApplier.java | 87 ++++++++++++----------
.../pherf/rules/SequentialDateDataGenerator.java | 68 +++++++++++++++++
.../pherf/rules/SequentialListDataGenerator.java | 66 ++++++++++++++++
.../rules/SequentialVarcharDataGenerator.java | 75 +++++++++++++++++++
.../tenantoperation/IdleTimeOperationSupplier.java | 3 +-
.../PreScenarioOperationSupplier.java | 33 ++++----
.../mt/tenantoperation/QueryOperationSupplier.java | 16 +++-
.../TenantOperationEventGenerator.java | 4 +-
.../mt/tenantoperation/TenantOperationFactory.java | 5 +-
.../tenantoperation/UpsertOperationSupplier.java | 16 +++-
.../UserDefinedOperationSupplier.java | 2 +
.../phoenix/pherf/ConfigurationParserTest.java | 27 ++++++-
.../apache/phoenix/pherf/RuleGeneratorTest.java | 5 +-
.../rules/SequentialDateDataGeneratorTest.java | 78 +++++++++++++++++++
.../rules/SequentialListDataGeneratorTest.java | 86 +++++++++++++++++++++
.../rules/SequentialVarcharDataGeneratorTest.java | 68 +++++++++++++++++
.../src/test/resources/datamodel/test_schema.sql | 3 +-
.../scenario/malicious_scenario_with_dtd.xml | 2 +-
.../src/test/resources/scenario/test_scenario.xml | 21 +++++-
.../scenario/test_workload_with_load_profile.xml | 81 +++++++++++++++-----
26 files changed, 703 insertions(+), 105 deletions(-)
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 94e82e3..5d2204d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -955,7 +955,7 @@ public abstract class BaseTest {
String fullTableName = SchemaUtil.getEscapedTableName(
rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM),
rs.getString(PhoenixDatabaseMetaData.TABLE_NAME));
- String ddl = "DROP " +
rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE) + " " + fullTableName;
+ String ddl = "DROP " +
rs.getString(PhoenixDatabaseMetaData.TABLE_TYPE) + " " + fullTableName + "
CASCADE";
String tenantId = rs.getString(1);
if (tenantId != null && !tenantId.equals(lastTenantId)) {
if (lastTenantId != null) {
diff --git
a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
index 75d6f6b..f3d7cba 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
@@ -30,7 +30,6 @@ import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.result.ResultUtil;
import org.apache.phoenix.pherf.schema.SchemaReader;
import org.apache.phoenix.pherf.util.PhoenixUtil;
-import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.After;
import org.junit.AfterClass;
@@ -39,8 +38,8 @@ import org.junit.experimental.categories.Category;
@Category(NeedsOwnMiniClusterTest.class)
public class ResultBaseTestIT extends ParallelStatsDisabledIT {
- protected static final String matcherScenario =
".*scenario/.*test_scenario.*xml";
- protected static final String matcherSchema =
".*datamodel/.*test_schema.*sql";
+ protected static final String matcherScenario =
".*scenario/.*test_scenario.xml";
+ protected static final String matcherSchema =
".*datamodel/.*test_schema.sql";
protected static PhoenixUtil util = PhoenixUtil.create(true);
protected static Properties properties;
diff --git
a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java
b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java
index bcdbdca..f200381 100644
---
a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java
+++
b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/MultiTenantOperationBaseIT.java
@@ -19,6 +19,8 @@
package org.apache.phoenix.pherf.workload.mt.tenantoperation;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.pherf.PherfConstants;
import org.apache.phoenix.pherf.XMLConfigParserTest;
@@ -26,7 +28,11 @@ import org.apache.phoenix.pherf.configuration.DataModel;
import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.schema.SchemaReader;
import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.junit.After;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -37,6 +43,7 @@ import java.util.Properties;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+@Category(NeedsOwnMiniClusterTest.class)
public class MultiTenantOperationBaseIT extends ParallelStatsDisabledIT {
enum TestOperationGroup {
upsertOp, queryOp1, queryOp2, idleOp, udfOp
@@ -76,4 +83,8 @@ public class MultiTenantOperationBaseIT extends
ParallelStatsDisabledIT {
return XMLConfigParser.readDataModel(p);
}
+ @AfterClass public static synchronized void tearDown() throws Exception {
+ dropNonSystemTables();
+ }
+
}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
index 3af8c3f..b60508d 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
@@ -23,7 +23,8 @@ import javax.xml.bind.annotation.XmlAttribute;
public class Ddl {
private String statement;
private String tableName;
-
+ private boolean useGlobalConnection;
+
public Ddl() {
}
@@ -55,7 +56,16 @@ public class Ddl {
public void setTableName(String tableName) {
this.tableName = tableName;
}
-
+
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
+
public String toString(){
if (statement.contains("?")) {
return statement.replace("?", tableName);
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
index 3116244..c66bbee 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
@@ -24,8 +24,8 @@ import java.util.List;
@XmlType
public class LoadProfile {
private static final int MIN_BATCH_SIZE = 1;
- private static final String DEFAULT_TENANT_ID_FMT = "00D%s%07d";
- private static final int DEFAULT_GROUP_ID_LEN = 5;
+ private static final String DEFAULT_TENANT_ID_FMT = "T%s%08d";
+ private static final int DEFAULT_GROUP_ID_LEN = 6;
private static final int DEFAULT_TENANT_ID_LEN = 15;
// Holds the batch size to be used in upserts.
@@ -37,7 +37,15 @@ public class LoadProfile {
* TenantId format should typically have 2 parts -
* 1. string fmt - that hold the tenant group id.
* 2. int fmt - that holds a random number between 1 and max tenants
- * for e.g DEFAULT_TENANT_ID_FMT = "00D%s%07d";
+ * for e.g DEFAULT_TENANT_ID_FMT = "T%s%08d";
+ *
+ * When the Tenant Group is configured to use a global connection,
+ * for now this is modelled as a special tenant whose id will translate to
"TGLOBAL00000001"
+ * since the group id => "GLOBAL" and num tenants = 1.
+ * For now this is a hack/temporary workaround.
+ *
+ * TODO :
+ * Ideally it needs to be built into the framework and injected during
event generation.
*/
private String tenantIdFormat;
private int groupIdLength;
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
index 0656917..d066cd5 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
@@ -21,13 +21,15 @@ package org.apache.phoenix.pherf.configuration;
import javax.xml.bind.annotation.XmlAttribute;
public class TenantGroup {
+ public static final String DEFAULT_GLOBAL_ID = "GLOBAL";
private String id;
private int weight;
private int numTenants;
+ private boolean useGlobalConnection;
@XmlAttribute
public String getId() {
- return id;
+ return useGlobalConnection ? DEFAULT_GLOBAL_ID: id;
}
public void setId(String id) {
@@ -36,7 +38,7 @@ public class TenantGroup {
@XmlAttribute
public int getWeight() {
- return weight;
+ return useGlobalConnection ? 100 : weight;
}
public void setWeight(int weight) {
@@ -44,9 +46,17 @@ public class TenantGroup {
}
@XmlAttribute
- public int getNumTenants() { return numTenants; }
+ public int getNumTenants() { return useGlobalConnection ? 1 : numTenants; }
public void setNumTenants(int numTenants) { this.numTenants = numTenants; }
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
index aeb3ec5..481604f 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
@@ -35,14 +35,12 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.text.SimpleDateFormat;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
public class RulesApplier {
private static final Logger LOGGER =
LoggerFactory.getLogger(RulesApplier.class);
- private static final AtomicLong COUNTER = new AtomicLong(0);
// Used to bail out of random distribution if it takes too long
// This should never happen when distributions add up to 100
@@ -59,7 +57,7 @@ public class RulesApplier {
private String cachedScenarioOverrideName;
private Map<DataTypeMapping, List> scenarioOverrideMap;
- private Map<Column,RuleBasedDataGenerator> columnRuleBasedDataGeneratorMap
= new HashMap<>();
+ private ConcurrentHashMap<String,RuleBasedDataGenerator>
columnRuleBasedDataGeneratorMap = new ConcurrentHashMap<>();
// Since rules are only relevant for a given data model,
// added a constructor to support a single data model =>
RulesApplier(DataModel model)
@@ -165,7 +163,6 @@ public class RulesApplier {
// Assume the first rule map
Map<DataTypeMapping, List> ruleMap = modelList.get(0);
List<Column> ruleList = ruleMap.get(phxMetaColumn.getType());
- //LOGGER.info(String.format("Did not found a correct override
column rule, %s, %s", phxMetaColumn.getName(), phxMetaColumn.getType()));
// Make sure Column from Phoenix Metadata matches a rule column
if (ruleList != null && ruleList.contains(phxMetaColumn)) {
@@ -175,9 +172,9 @@ public class RulesApplier {
value = getDataValue(columnRule);
} else {
- LOGGER.warn("Attempted to apply rule to data, but could not
find a rule to match type:"
- + phxMetaColumn.getType()
- );
+ LOGGER.warn(String.format("Attempted to apply rule to data, "
+ + "but could not find a rule to match type %s on %s",
+ phxMetaColumn.getType(), phxMetaColumn.getName()));
}
}
@@ -191,7 +188,7 @@ public class RulesApplier {
* @param column {@link org.apache.phoenix.pherf.configuration.Column}
Column rule to get data for
* @return {@link org.apache.phoenix.pherf.rules.DataValue} {@code
Container Type --> Value mapping }
*/
- public DataValue getDataValue(Column column) throws Exception{
+ public DataValue getDataValue(Column column) throws Exception {
DataValue data = null;
String prefix = "";
int length = column.getLength();
@@ -218,15 +215,14 @@ public class RulesApplier {
case VARBINARY:
case CHAR:
// Use the specified data values from configs if they exist
- if ((column.getDataValues() != null) &&
(column.getDataValues().size() > 0)) {
+ if (DataSequence.SEQUENTIAL.equals(column.getDataSequence())) {
+ RuleBasedDataGenerator generator =
getRuleBasedDataGeneratorForColumn(column);
+ data = generator.getDataValue();
+ } else if ((column.getDataValues() != null) &&
(column.getDataValues().size() > 0)) {
data = pickDataValueFromList(dataValues);
} else {
Preconditions.checkArgument(length > 0, "length needs to
be > 0");
- if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
- data = getSequentialVarcharDataValue(column);
- } else {
- data = getRandomDataValue(column);
- }
+ data = getRandomDataValue(column);
}
break;
case VARCHAR_ARRAY:
@@ -297,6 +293,9 @@ public class RulesApplier {
data = pickDataValueFromList(dataValues);
// Check if date has right format or not
data.setValue(checkDatePattern(data.getValue()));
+ } else
if(DataSequence.SEQUENTIAL.equals(column.getDataSequence())) {
+ RuleBasedDataGenerator generator =
getRuleBasedDataGeneratorForColumn(column);
+ data = generator.getDataValue();
} else if (column.getUseCurrentDate() != true){
int minYear = (int) column.getMinValue();
int maxYear = (int) column.getMaxValue();
@@ -537,28 +536,6 @@ public class RulesApplier {
return ruleAppliedColumn;
}
- /**
- * Add a numerically increasing counter onto the and of a random string.
- * Incremented counter should be thread safe.
- *
- * @param column {@link org.apache.phoenix.pherf.configuration.Column}
- * @return {@link org.apache.phoenix.pherf.rules.DataValue}
- */
- private DataValue getSequentialVarcharDataValue(Column column) {
- DataValue data = null;
- long inc = COUNTER.getAndIncrement();
- String strInc = String.valueOf(inc);
- int paddedLength = column.getLengthExcludingPrefix();
- String strInc1 = StringUtils.leftPad(strInc, paddedLength, "0");
- String strInc2 = StringUtils.right(strInc1,
column.getLengthExcludingPrefix());
- String varchar = (column.getPrefix() != null) ? column.getPrefix() +
strInc2:
- strInc2;
-
- // Truncate string back down if it exceeds length
- varchar = StringUtils.left(varchar,column.getLength());
- data = new DataValue(column.getType(), varchar);
- return data;
- }
private DataValue getRandomDataValue(Column column) {
String varchar =
RandomStringUtils.randomAlphanumeric(column.getLength());
@@ -570,11 +547,39 @@ public class RulesApplier {
}
private RuleBasedDataGenerator getRuleBasedDataGeneratorForColumn(Column
column) {
- RuleBasedDataGenerator generator =
columnRuleBasedDataGeneratorMap.get(column);
+ RuleBasedDataGenerator generator =
columnRuleBasedDataGeneratorMap.get(column.getName());
if(generator == null) {
- //For now we only have one of these, likely this should replace
all all the methods
- generator = new SequentialIntegerDataGenerator(column);
- columnRuleBasedDataGeneratorMap.put(column,generator);
+ //For now we only have couple of these, likely this should replace
for all the methods
+ switch (column.getType()) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ if ((column.getDataValues() != null) &&
(column.getDataValues().size() > 0)) {
+ generator = new SequentialListDataGenerator(column);
+ } else {
+ generator = new SequentialVarcharDataGenerator(column);
+ }
+ break;
+ case DATE:
+ case TIMESTAMP:
+ generator = new SequentialDateDataGenerator(column);
+ break;
+ case BIGINT:
+ case INTEGER:
+ case TINYINT:
+ case UNSIGNED_LONG:
+ generator = new SequentialIntegerDataGenerator(column);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format("No rule based generator supported for
column type %s on %s",
+ column.getType(), column.getName()));
+ }
+ RuleBasedDataGenerator oldGenerator =
columnRuleBasedDataGeneratorMap.putIfAbsent(column.getName(),generator);
+ if (oldGenerator != null) {
+ // Another thread succeeded in registering their generator
first, so let's use that.
+ generator = oldGenerator;
+ }
}
return generator;
}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java
new file mode 100644
index 0000000..63328d0
--- /dev/null
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A generator for sequentially increasing dates.
+ * For now the increments are fixed at 1 second.
+ */
+public class SequentialDateDataGenerator implements RuleBasedDataGenerator {
+ private static DateTimeFormatter FMT =
DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ private final Column columnRule;
+ private final AtomicInteger counter;
+ private final LocalDateTime startDateTime = new LocalDateTime();
+
+ public SequentialDateDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() ==
DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(isDateType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicInteger(0);
+ }
+
+ /**
+ * Note that this method rolls over for attempts to get larger than
maxValue
+ * @return new DataValue
+ */
+ @Override
+ public DataValue getDataValue() {
+ LocalDateTime newDateTime =
startDateTime.plusSeconds(counter.getAndIncrement());
+ String formattedDateTime = newDateTime.toString(FMT);
+ return new DataValue(columnRule.getType(), formattedDateTime);
+ }
+
+ boolean isDateType(DataTypeMapping mapping) {
+ switch (mapping) {
+ case DATE:
+ case TIMESTAMP:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java
new file mode 100644
index 0000000..ddcc354
--- /dev/null
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A generator to round robin thru a list of values.
+ */
+
+public class SequentialListDataGenerator implements RuleBasedDataGenerator {
+ private final Column columnRule;
+ private final AtomicLong counter;
+
+ public SequentialListDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() ==
DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(columnRule.getDataValues().size() > 0);
+ Preconditions.checkArgument(isAllowedType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicLong(0);
+ }
+
+ /**
+ * Note that this method rolls over for attempts to get larger than
maxValue
+ * @return new DataValue
+ */
+ @Override
+ public DataValue getDataValue() {
+ long pos = counter.getAndIncrement();
+ int index = (int) pos % columnRule.getDataValues().size();
+ return columnRule.getDataValues().get(index);
+ }
+
+ boolean isAllowedType(DataTypeMapping mapping) {
+ // For now only varchar list are supported
+ switch (mapping) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java
new file mode 100644
index 0000000..08bc1dd
--- /dev/null
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A generator for sequentially increasing varchar values.
+ */
+public class SequentialVarcharDataGenerator implements RuleBasedDataGenerator {
+ private final Column columnRule;
+ private final AtomicLong counter;
+
+ public SequentialVarcharDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() ==
DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(isVarcharType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicLong(0);
+ }
+
+ /**
+ * Add a numerically increasing counter onto the and of a random string.
+ * Incremented counter should be thread safe.
+ *
+ * @return {@link org.apache.phoenix.pherf.rules.DataValue}
+ */
+ @Override
+ public DataValue getDataValue() {
+ DataValue data = null;
+ long inc = counter.getAndIncrement();
+ String strInc = String.valueOf(inc);
+ int paddedLength = columnRule.getLengthExcludingPrefix();
+ String strInc1 = StringUtils.leftPad(strInc, paddedLength, "x");
+ String strInc2 = StringUtils.right(strInc1,
columnRule.getLengthExcludingPrefix());
+ String varchar = (columnRule.getPrefix() != null) ?
columnRule.getPrefix() + strInc2:
+ strInc2;
+
+ // Truncate string back down if it exceeds length
+ varchar = StringUtils.left(varchar,columnRule.getLength());
+ data = new DataValue(columnRule.getType(), varchar);
+ return data;
+ }
+
+ boolean isVarcharType(DataTypeMapping mapping) {
+ switch (mapping) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java
index ab45c27..c65f430 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/IdleTimeOperationSupplier.java
@@ -25,6 +25,7 @@ import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.workload.mt.IdleTimeOperation;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,7 @@ class IdleTimeOperationSupplier extends BaseOperationSupplier
{
@Override
public OperationStats apply(final TenantOperationInfo input) {
-
+ Preconditions.checkNotNull(input);
final IdleTimeOperation operation = (IdleTimeOperation)
input.getOperation();
final IdleTime idleTime = operation.getIdleTime();
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java
index 4f1e3e3..a323a36 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/PreScenarioOperationSupplier.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.pherf.workload.mt.tenantoperation;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
import org.apache.phoenix.thirdparty.com.google.common.base.Function;
import org.apache.phoenix.pherf.configuration.DataModel;
import org.apache.phoenix.pherf.configuration.Ddl;
@@ -25,6 +26,7 @@ import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
import org.apache.phoenix.pherf.workload.mt.PreScenarioOperation;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,32 +50,37 @@ class PreScenarioOperationSupplier extends
BaseOperationSupplier {
@Override
public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
final PreScenarioOperation operation = (PreScenarioOperation)
input.getOperation();
- final String tenantId = input.getTenantId();
final String tenantGroup = input.getTenantGroupId();
final String opGroup = input.getOperationGroupId();
final String tableName = input.getTableName();
final String scenarioName = input.getScenarioName();
- final String opName = String.format("%s:%s:%s:%s:%s",
- scenarioName, tableName, opGroup, tenantGroup,
tenantId);
+ final boolean isTenantGroupGlobal =
(tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
long startTime = EnvironmentEdgeManager.currentTimeMillis();
int status = 0;
if (!operation.getPreScenarioDdls().isEmpty()) {
- try (Connection conn =
phoenixUtil.getConnection(tenantId)) {
- for (Ddl ddl : operation.getPreScenarioDdls()) {
- LOGGER.info("\nExecuting DDL:" + ddl + " on
tenantId:" + tenantId);
- phoenixUtil.executeStatement(ddl.toString(), conn);
+ for (Ddl ddl : operation.getPreScenarioDdls()) {
+ // TODO:
+ // Ideally the fact that the op needs to executed
using global connection
+ // needs to be built into the framework and injected
during event generation.
+ // For now a special tenant whose id =
"TGLOBAL00000001" will be logged.
+ final String tenantId = isTenantGroupGlobal ||
ddl.isUseGlobalConnection() ? null : input.getTenantId();
+ final String opName = String.format("%s:%s:%s:%s:%s",
+ scenarioName, tableName, opGroup, tenantGroup,
input.getTenantId());
+
+ try (Connection conn =
phoenixUtil.getConnection(tenantId)) {
+ LOGGER.info("\nExecuting DDL:" + ddl + ",
OPERATION:" + opName);
+ String sql = ddl.toString();
+ phoenixUtil.executeStatement(sql, conn);
if
(ddl.getStatement().toUpperCase().contains(phoenixUtil.ASYNC_KEYWORD)) {
phoenixUtil.waitForAsyncIndexToFinish(ddl.getTableName());
}
+ } catch (Exception e) {
+ LOGGER.error("Operation " + opName + " failed with
exception ", e);
+ status = -1;
}
- } catch (SQLException sqle) {
- LOGGER.error("Operation " + opName + " failed with
exception ", sqle);
- status = -1;
- } catch (Exception e) {
- LOGGER.error("Operation " + opName + " failed with
exception ", e);
- status = -1;
}
}
long totalDuration =
EnvironmentEdgeManager.currentTimeMillis() - startTime;
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java
index 88eec68..99188ce 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/QueryOperationSupplier.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.pherf.workload.mt.tenantoperation;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
import org.apache.phoenix.thirdparty.com.google.common.base.Function;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.pherf.configuration.DataModel;
@@ -26,6 +27,7 @@ import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
import org.apache.phoenix.pherf.workload.mt.QueryOperation;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,17 +52,23 @@ public class QueryOperationSupplier extends
BaseOperationSupplier {
@Override
public OperationStats apply(final TenantOperationInfo input) {
-
+ Preconditions.checkNotNull(input);
final QueryOperation operation = (QueryOperation)
input.getOperation();
+ final Query query = operation.getQuery();
final String tenantGroup = input.getTenantGroupId();
final String opGroup = input.getOperationGroupId();
- final String tenantId = input.getTenantId();
final String scenarioName = input.getScenarioName();
final String tableName = input.getTableName();
- final Query query = operation.getQuery();
+
+ // TODO:
+ // Ideally the fact that the op needs to executed using global
connection
+ // needs to be built into the framework and injected during
event generation.
+ // For now a special tenant whose id = "TGLOBAL00000001" will
be logged.
+ final boolean isTenantGroupGlobal =
(tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
+ final String tenantId = isTenantGroupGlobal ||
query.isUseGlobalConnection() ? null : input.getTenantId();
String opName = String.format("%s:%s:%s:%s:%s", scenarioName,
tableName,
- opGroup, tenantGroup, tenantId);
+ opGroup, tenantGroup, input.getTenantId());
LOGGER.info("\nExecuting query " + query.getStatement());
long startTime = 0;
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java
index 676c510..de8f95a 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationEventGenerator.java
@@ -120,10 +120,10 @@ public class TenantOperationEventGenerator
Operation op = operationMap.get(opId);
int numTenants = tenantGroupMap.get(tenantGroupId).getNumTenants();
- String tenantIdPrefix = Strings.padStart(tenantGroupId,
loadProfile.getGroupIdLength(), '0');
+ String tenantIdPrefix = Strings.padStart(tenantGroupId,
loadProfile.getGroupIdLength(), 'x');
String formattedTenantId =
String.format(loadProfile.getTenantIdFormat(),
tenantIdPrefix.substring(0,
loadProfile.getGroupIdLength()), RANDOM.nextInt(numTenants));
- String paddedTenantId = Strings.padStart(formattedTenantId,
loadProfile.getTenantIdLength(), '0');
+ String paddedTenantId = Strings.padStart(formattedTenantId,
loadProfile.getTenantIdLength(), 'x');
String tenantId = paddedTenantId.substring(0,
loadProfile.getTenantIdLength());
TenantOperationInfo sample = new TenantOperationInfo(modelName,
scenarioName, tableName,
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java
index 365984f..6e86e14 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/TenantOperationFactory.java
@@ -43,6 +43,7 @@ import org.apache.phoenix.pherf.workload.mt.EventGenerator;
import org.apache.phoenix.pherf.workload.mt.IdleTimeOperation;
import org.apache.phoenix.pherf.workload.mt.Operation;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
+import org.apache.phoenix.pherf.workload.mt.PreScenarioOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -274,8 +275,8 @@ public class TenantOperationFactory {
// Check if the scenario has a PRE_RUN operation.
if (preRunOpSupplier != null) {
// Initialize the tenant using the pre scenario ddls.
- final org.apache.phoenix.pherf.workload.mt.PreScenarioOperation
- operation = new
org.apache.phoenix.pherf.workload.mt.PreScenarioOperation() {
+ final PreScenarioOperation
+ operation = new PreScenarioOperation() {
@Override public List<Ddl> getPreScenarioDdls() {
List<Ddl> ddls = scenario.getPreScenarioDdls();
return ddls == null ? Lists.<Ddl>newArrayList() : ddls;
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java
index 5b25c12..30f953b 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UpsertOperationSupplier.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.pherf.workload.mt.tenantoperation;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
import org.apache.phoenix.thirdparty.com.google.common.base.Function;
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.DataModel;
@@ -26,6 +27,7 @@ import org.apache.phoenix.pherf.configuration.Upsert;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
import org.apache.phoenix.pherf.workload.mt.UpsertOperation;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,22 +54,28 @@ class UpsertOperationSupplier extends BaseOperationSupplier
{
@Override
public OperationStats apply(final TenantOperationInfo input) {
-
+ Preconditions.checkNotNull(input);
final int batchSize = loadProfile.getBatchSize();
final boolean useBatchApi = batchSize != 0;
final int rowCount = useBatchApi ? batchSize : 1;
final UpsertOperation operation = (UpsertOperation)
input.getOperation();
+ final Upsert upsert = operation.getUpsert();
final String tenantGroup = input.getTenantGroupId();
final String opGroup = input.getOperationGroupId();
- final String tenantId = input.getTenantId();
- final Upsert upsert = operation.getUpsert();
final String tableName = input.getTableName();
final String scenarioName = input.getScenarioName();
final List<Column> columns = upsert.getColumn();
+ // TODO:
+ // Ideally the fact that the op needs to executed using global
connection
+ // needs to be built into the framework and injected during
event generation.
+ // For now a special tenant whose id = "TGLOBAL00000001" will
be logged.
+
+ final boolean isTenantGroupGlobal =
(tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
+ final String tenantId = isTenantGroupGlobal ||
upsert.isUseGlobalConnection() ? null : input.getTenantId();
final String opName = String.format("%s:%s:%s:%s:%s",
- scenarioName, tableName, opGroup, tenantGroup,
tenantId);
+ scenarioName, tableName, opGroup, tenantGroup,
input.getTenantId());
long rowsCreated = 0;
long startTime = 0, duration, totalDuration;
diff --git
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java
index ae8ce6f..f1f650a 100644
---
a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java
+++
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/tenantoperation/UserDefinedOperationSupplier.java
@@ -24,6 +24,7 @@ import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.workload.mt.OperationStats;
import org.apache.phoenix.pherf.workload.mt.UserDefinedOperation;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
/**
@@ -40,6 +41,7 @@ class UserDefinedOperationSupplier extends
BaseOperationSupplier {
return new Function<TenantOperationInfo, OperationStats>() {
@Override
public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
// TODO : implement user defined operation invocation.
long startTime = EnvironmentEdgeManager.currentTimeMillis();
long duration = EnvironmentEdgeManager.currentTimeMillis() -
startTime;
diff --git
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
index 929b7fa..78f50a6 100644
---
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
+++
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Set;
+import org.apache.phoenix.pherf.rules.RulesApplier;
import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import org.apache.phoenix.pherf.configuration.*;
import org.apache.phoenix.pherf.rules.DataValue;
@@ -162,6 +163,30 @@ public class ConfigurationParserTest extends
ResultBaseTest {
1, testScenarioWithLoadProfile.getQuerySet().size());
assertEquals("#Queries within the first querySet is not as expected ",
2,
testScenarioWithLoadProfile.getQuerySet().get(0).getQuery().size());
+
+ // Test configuration for global connection
+ Scenario testScenarioWithGlobalConn = scenarioList.get(2);
+ LoadProfile loadProfileWithGlobalConn =
testScenarioWithGlobalConn.getLoadProfile();
+ assertEquals("batch size not as expected: ",
+ 1, loadProfileWithGlobalConn.getBatchSize());
+ assertEquals("num operations not as expected: ",
+ 1000, loadProfileWithGlobalConn.getNumOperations());
+ assertEquals("tenant group size is not as expected: ",
+ 1, loadProfileWithGlobalConn.getTenantDistribution().size());
+ assertEquals("global tenant is not as expected: ",
+ 1,
loadProfileWithGlobalConn.getTenantDistribution().get(0).getNumTenants());
+ assertEquals("global tenant id is not as expected: ",
+ "GLOBAL",
loadProfileWithGlobalConn.getTenantDistribution().get(0).getId());
+ assertEquals("global tenant weight is not as expected: ",
+ 100,
loadProfileWithGlobalConn.getTenantDistribution().get(0).getWeight());
+ assertEquals("operation group size is not as expected: ",
+ 1,loadProfileWithGlobalConn.getOpDistribution().size());
+ assertEquals("UpsertSet size is not as expected ",
+ 1, testScenarioWithGlobalConn.getUpserts().size());
+ assertEquals("#Column within the first upsert is not as expected ",
+ 7,
testScenarioWithGlobalConn.getUpserts().get(0).getColumn().size());
+ assertEquals("Upsert operation not using global connection as expected
",
+ true,
testScenarioWithGlobalConn.getUpserts().get(0).isUseGlobalConnection());
}
private URL getResourceUrl(String resourceName) {
@@ -186,7 +211,7 @@ public class ConfigurationParserTest extends ResultBaseTest
{
private void assertDateValue(List<Column> dataMappingColumns) {
for (Column dataMapping : dataMappingColumns) {
if ((dataMapping.getType() == DataTypeMapping.DATE) &&
(dataMapping.getName()
- .equals("CREATED_DATE"))) {
+ .equals("SOME_DATE"))) {
// First rule should have min/max set
assertNotNull(dataMapping.getDataValues().get(0).getMinValue());
assertNotNull(dataMapping.getDataValues().get(0).getMaxValue());
diff --git
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
index c439d38..c62922b 100644
---
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
+++
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
@@ -59,7 +59,7 @@ public class RuleGeneratorTest {
RulesApplier rulesApplier = loader.getRulesApplier();
for (Column dataMapping : model.getDataMappingColumns()) {
- if ((dataMapping.getType() == DataTypeMapping.DATE) &&
(dataMapping.getName().equals("CREATED_DATE"))) {
+ if ((dataMapping.getType() == DataTypeMapping.DATE) &&
(dataMapping.getName().equals("SOME_DATE"))) {
// Test directly through generator method and that it converts
to Phoenix type
assertRandomDateValue(dataMapping, rulesApplier);
@@ -92,7 +92,8 @@ public class RuleGeneratorTest {
for (Column dataMapping : model.getDataMappingColumns()) {
if ((dataMapping.getType() == DataTypeMapping.DATE)
- && (dataMapping.getUseCurrentDate() == true)) {
+ && (dataMapping.getUseCurrentDate() == true)
+ && (dataMapping.getDataSequence() !=
DataSequence.SEQUENTIAL)) {
// Generate the date using rules
DataValue value = rulesApplier.getDataValue(dataMapping);
diff --git
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java
new file mode 100644
index 0000000..3cc75e4
--- /dev/null
+++
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.junit.Test;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class SequentialDateDataGeneratorTest {
+ SequentialDateDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialDateDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonDate() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ //should reject this Column
+ generator = new SequentialDateDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd
HH:mm:ss.SSS");
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ LocalDateTime startDateTime = new LocalDateTime();
+
+ // The increments are the of 1 sec units
+ generator = new SequentialDateDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ LocalDateTime result1LocalTime =
FMT.parseDateTime(result1.getValue()).toLocalDateTime();
+ assertFalse(result1LocalTime.isBefore(startDateTime));
+ DataValue result2 = generator.getDataValue();
+ LocalDateTime result2LocalTime =
FMT.parseDateTime(result2.getValue()).toLocalDateTime();
+ assertEquals(result2LocalTime.minusSeconds(1), result1LocalTime);
+ DataValue result3 = generator.getDataValue();
+ LocalDateTime result3LocalTime =
FMT.parseDateTime(result3.getValue()).toLocalDateTime();
+ assertEquals(result3LocalTime.minusSeconds(1), result2LocalTime);
+ DataValue result4 = generator.getDataValue();
+ LocalDateTime result4LocalTime =
FMT.parseDateTime(result4.getValue()).toLocalDateTime();
+ assertEquals(result4LocalTime.minusSeconds(1), result3LocalTime);
+ }
+}
diff --git
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java
new file mode 100644
index 0000000..d590167
--- /dev/null
+++
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+
+public class SequentialListDataGeneratorTest {
+ SequentialListDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialListDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonVarchar() {
+ DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd
HH:mm:ss.SSS");
+ LocalDateTime startDateTime = new LocalDateTime();
+ String formattedDateTime = startDateTime.toString(FMT);
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ List<DataValue> values = new ArrayList<>();
+ values.add(new DataValue(DATE, formattedDateTime));
+ values.add(new DataValue(DATE, formattedDateTime));
+ values.add(new DataValue(DATE, formattedDateTime));
+ columnA.setDataValues(values);
+
+ //should reject this Column
+ generator = new SequentialListDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ List<DataValue> values = new ArrayList<>();
+ values.add(new DataValue(VARCHAR, "A"));
+ values.add(new DataValue(VARCHAR, "B"));
+ values.add(new DataValue(VARCHAR, "C"));
+ columnA.setDataValues(values);
+
+ generator = new SequentialListDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ assertEquals("A", result1.getValue());
+ DataValue result2 = generator.getDataValue();
+ assertEquals("B", result2.getValue());
+ DataValue result3 = generator.getDataValue();
+ assertEquals("C", result3.getValue());
+ DataValue result4 = generator.getDataValue();
+ assertEquals("A", result4.getValue());
+ }
+}
diff --git
a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java
new file mode 100644
index 0000000..0157721
--- /dev/null
+++
b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.junit.Test;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.INTEGER;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+
+public class SequentialVarcharDataGeneratorTest {
+ SequentialVarcharDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialVarcharDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonVarchar() {
+ Column columnA = new Column();
+ columnA.setType(INTEGER);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ //should reject this Column
+ generator = new SequentialVarcharDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setLength(15);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ generator = new SequentialVarcharDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx0", result1.getValue());
+ DataValue result2 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx1", result2.getValue());
+ DataValue result3 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx2", result3.getValue());
+ DataValue result4 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx3", result4.getValue());
+ }
+}
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
index a5a7274..9a1c856 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
@@ -19,7 +19,8 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
TENANT_ID CHAR(15) NOT NULL,
PARENT_ID CHAR(15) NOT NULL,
CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
+ SOME_DATE DATE,
+ RND_DATE DATE,
TS_DATE TIMESTAMP,
PRESENT_DATE DATE,
OTHER_ID CHAR(15),
diff --git
a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
index e8528f3..cc45395 100644
--- a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
+++ b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
@@ -40,7 +40,7 @@
least once.
-->
<scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno"
- ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1
VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE"
+ ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1
VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE"
rowCount="100" name="testMTDdlWriteScenario">
</scenario>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index 853b857..db3caac 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -59,13 +59,14 @@
<column>
<type>DATE</type>
<!--SEQUENTIAL is unsupported for DATE -->
+ <userDefined>true</userDefined>
<dataSequence>RANDOM</dataSequence>
<!-- Number [0-100] that represents the probability of creating a
null value -->
<!-- The higher the number, the more like the value will returned
will be null -->
<!-- Leaving this tag out is equivalent to having a 0 probability.
i.e. never null -->
<nullChance>0</nullChance>
<useCurrentDate>true</useCurrentDate>
- <name>NOW_DATE</name>
+ <name>RND_DATE</name>
</column>
<column>
<type>DECIMAL</type>
@@ -94,7 +95,20 @@
</column>
<column>
<type>DATE</type>
+ <userDefined>true</userDefined>
+ <!--SEQUENTIAL is now supported for DATE -->
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a
null value -->
+ <!-- The higher the number, the more like the value will returned
will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability.
i.e. never null -->
+ <nullChance>0</nullChance>
+ <useCurrentDate>true</useCurrentDate>
<name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <userDefined>true</userDefined>
+ <name>SOME_DATE</name>
<minValue>1975</minValue>
<maxValue>2025</maxValue>
<valuelist>
@@ -115,6 +129,7 @@
</column>
<column>
<type>DATE</type>
+ <userDefined>true</userDefined>
<name>PRESENT_DATE</name>
<minValue>1975</minValue>
<maxValue>2025</maxValue>
@@ -331,7 +346,7 @@
<scenario tableName="PHERF.TEST_VIEW" tenantId="xyzdefghijklmno"
rowCount="100" name="testMTWriteScenario">
<preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW
(field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM
PHERF.TEST_MULTI_TENANT_TABLE" />
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW
(field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
</preScenarioDdls>
</scenario>
<!-- Scenario level DDL that is dynamically executed before the Write
Workload is run.
@@ -344,7 +359,7 @@
<scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno"
rowCount="100" name="testMTDdlWriteScenario">
<preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW
(field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" />
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW
(field1 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
</preScenarioDdls>
</scenario>
diff --git
a/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
index 855c1fe..3a1c710 100644
---
a/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
+++
b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
@@ -234,14 +234,14 @@
<loadProfile>
<batchSize>1</batchSize>
<numOperations>1000</numOperations>
- <tenantDistribution id="t111" weight="10"
numTenants="10"></tenantDistribution>
- <tenantDistribution id="t112" weight="10"
numTenants="10"></tenantDistribution>
- <tenantDistribution id="t113" weight="80"
numTenants="1"></tenantDistribution>
- <opDistribution id="op111" weight="50"></opDistribution>
- <opDistribution id="op112" weight="0"></opDistribution>
- <opDistribution id="op113" weight="0"></opDistribution>
- <opDistribution id="op114" weight="50"></opDistribution>
- <opDistribution id="op115" weight="0"></opDistribution>
+ <tenantDistribution id="t111" weight="10" numTenants="10"/>
+ <tenantDistribution id="t112" weight="10" numTenants="10"/>
+ <tenantDistribution id="t113" weight="80" numTenants="1"/>
+ <opDistribution id="op111" weight="50"/>
+ <opDistribution id="op112" weight="0"/>
+ <opDistribution id="op113" weight="0"/>
+ <opDistribution id="op114" weight="50"/>
+ <opDistribution id="op115" weight="0"/>
</loadProfile>
@@ -283,7 +283,7 @@
</upserts>
<idleTimes>
- <idleTime id="op114" idleTime="50"></idleTime>
+ <idleTime id="op114" idleTime="50"/>
</idleTimes>
<udfs>
<udf id="op115" >
@@ -302,14 +302,14 @@
<loadProfile>
<batchSize>5</batchSize>
<numOperations>1000</numOperations>
- <tenantDistribution id="t121" weight="10"
numTenants="5"></tenantDistribution>
- <tenantDistribution id="t122" weight="10"
numTenants="5"></tenantDistribution>
- <tenantDistribution id="t123" weight="80"
numTenants="5"></tenantDistribution>
- <opDistribution id="op121" weight="50"></opDistribution>
- <opDistribution id="op122" weight="5"></opDistribution>
- <opDistribution id="op123" weight="5"></opDistribution>
- <opDistribution id="op124" weight="40"></opDistribution>
- <opDistribution id="op125" weight="0"></opDistribution>
+ <tenantDistribution id="t121" weight="10" numTenants="5"/>
+ <tenantDistribution id="t122" weight="10" numTenants="5"/>
+ <tenantDistribution id="t123" weight="80" numTenants="5"/>
+ <opDistribution id="op121" weight="50"/>
+ <opDistribution id="op122" weight="5"/>
+ <opDistribution id="op123" weight="5"/>
+ <opDistribution id="op124" weight="40"/>
+ <opDistribution id="op125" weight="0"/>
</loadProfile>
<preScenarioDdls>
@@ -350,7 +350,7 @@
</upserts>
<idleTimes>
- <idleTime id="op124" idleTime="100"></idleTime>
+ <idleTime id="op124" idleTime="100"/>
</idleTimes>
<querySet>
<query id="op122" statement="select count(*) from PHERF.Z12"/>
@@ -365,6 +365,51 @@
</udfs>
</scenario>
+ <scenario tableName="PHERF.Z13" name="scenario_13">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <tenantDistribution useGlobalConnection="true"/>
+ <opDistribution id="op131" weight="100"/>
+ </loadProfile>
+
+ <preScenarioDdls>
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.Z13 (field1
VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE WHERE PARENT_ID =
'aAAyYhnNbBs9kWk'" />
+ </preScenarioDdls>
+
+ <upserts>
+ <upsert id="op131" useGlobalConnection="true">
+ <column>
+ <type>CHAR</type>
+ <name>PARENT_ID_SEQ</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OTHER_ID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OLDVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>NEWVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ </upsert>
+ </upserts>
+ </scenario>
</scenarios>
</datamodel>
