PHOENIX-2227 Added the ability to Pherf to define a DDL statement that will be executed before a scenario is run to support dynamically creating multi-tenant views we are going to write and read from
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/926bc72c Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/926bc72c Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/926bc72c Branch: refs/heads/master Commit: 926bc72c45f8e55c9767c2d3406a76a50f9eac32 Parents: ebebf58 Author: Jan <[email protected]> Authored: Thu Sep 3 17:48:18 2015 -0700 Committer: Cody Marcel <[email protected]> Committed: Wed Sep 9 16:10:59 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/pherf/DataIngestIT.java | 27 +++++++++++++++++--- .../phoenix/pherf/configuration/Scenario.java | 13 ++++++++++ .../apache/phoenix/pherf/util/PhoenixUtil.java | 21 +++++++++++++++ .../phoenix/pherf/workload/WriteWorkload.java | 5 +++- .../test/resources/scenario/test_scenario.xml | 21 ++++++++++++++- 5 files changed, 81 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/926bc72c/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java index 1defcf5..f2a1a07 100644 --- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java @@ -18,10 +18,6 @@ package org.apache.phoenix.pherf; -import com.jcabi.jdbc.JdbcSession; -import com.jcabi.jdbc.Outcome; - -import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -35,6 +31,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.jcabi.jdbc.JdbcSession; +import com.jcabi.jdbc.Outcome; +import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats; import org.apache.phoenix.pherf.configuration.Column; import org.apache.phoenix.pherf.configuration.DataModel; import org.apache.phoenix.pherf.configuration.DataTypeMapping; @@ -173,6 +172,26 @@ public class DataIngestIT extends ResultBaseTestIT { assertExpectedNumberOfRecordsWritten(scenario); } + + @Test + public void testMultiTenantScenarioRunBeforeWriteWorkload() throws Exception { + // Arrange + Scenario scenario = parser.getScenarioByName("testMTDdlWriteScenario"); + WorkloadExecutor executor = new WorkloadExecutor(); + executor.add(new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO)); + + // Act + try { + // Wait for data to load up. + executor.get(); + executor.shutdown(); + } catch (Exception e) { + fail("Failed to load data. An exception was thrown: " + e.getMessage()); + } + + assertExpectedNumberOfRecordsWritten(scenario); + } + private void assertExpectedNumberOfRecordsWritten(Scenario scenario) throws Exception, SQLException { Connection connection = util.getConnection(scenario.getTenantId()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/926bc72c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java index 6c949d8..200fdc5 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java @@ -39,6 +39,7 @@ public class Scenario { private WriteParams writeParams; private String name; private String tenantId; + private String ddl; public Scenario() { writeParams = new WriteParams(); @@ -178,6 +179,18 @@ public class Scenario { this.tenantId = tenantId; } + /** + * Scenario level DDL that is executed before running the scenario. + */ + @XmlAttribute + public String getDdl() { + return ddl; + } + + public void setDdl(String ddl) { + this.ddl = ddl; + } + public WriteParams getWriteParams() { return writeParams; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/926bc72c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java index fad06a1..57858a3 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java @@ -243,6 +243,27 @@ public class PhoenixUtil { } } } + + /** + * Executes any ddl defined at the scenario level. This is executed before we commence + * the data load. + * + * @throws Exception + */ + public void executeScenarioDdl(Scenario scenario) throws Exception { + if (null != scenario.getDdl()) { + Connection conn = null; + try { + logger.info("\nExecuting DDL:" + scenario.getDdl() + " on tenantId:" + + scenario.getTenantId()); + executeStatement(scenario.getDdl(), conn = getConnection(scenario.getTenantId())); + } finally { + if (null != conn) { + conn.close(); + } + } + } + } public static String getZookeeper() { return zookeeper; http://git-wip-us.apache.org/repos/asf/phoenix/blob/926bc72c/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java index 12962c7..b38f875 100644 --- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java +++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java @@ -164,7 +164,10 @@ public class WriteWorkload implements Workload { DataLoadThreadTime dataLoadThreadTime, Scenario scenario) throws Exception { logger.info("\nLoading " + scenario.getRowCount() + " rows for " + scenario.getTableName()); long start = System.currentTimeMillis(); - + + // Execute any Scenario DDL before running workload + pUtil.executeScenarioDdl(scenario); + List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario); waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches); http://git-wip-us.apache.org/repos/asf/phoenix/blob/926bc72c/phoenix-pherf/src/test/resources/scenario/test_scenario.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml index b5fe564..50a603e 100644 --- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml +++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml @@ -220,8 +220,27 @@ <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/> </querySet> </scenario> - <!-- Test writing to a Multi-tenant View --> + <!-- To configure a Write Workload to write to a tenant specific view users need to + specify the tenantId attribute on the scenario, specifying the tenant they + want to write data for as the attribute value. This tells Pherf to take out a + tenant-specific connection for executing the write workload. + The name of the tenant specific view to write to can then be specified as the value of + the tablename attribute. This assumes the tenant specific view has been created. To + dynamically create the view see comments below with regard to the ddl attribute. + --> <scenario tableName="PHERF.TEST_VIEW" tenantId="abcdefghijklmno" rowCount="100" name="testMTWriteScenario"> </scenario> + <!-- Scenario level DDL that is dynamically executed before the Write Workload is run. + This pattern is really useful when you want to write data to multi-tenant view and the tenant id is + tightly bound to the scenario. In such cases you can't create the view through the data model flow. + The value of the tableName attribute is name of the view that is dynamically created based on the DDL + in the ddl attribute. Queries accessing the View will need to manually make sure Pherf was run with the -l option at + least once. + --> + <scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno" + ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" + rowCount="100" name="testMTDdlWriteScenario"> + </scenario> + </scenarios> </datamodel> \ No newline at end of file
