Repository: phoenix Updated Branches: refs/heads/master bb30ed516 -> ee5b6707b
PHOENIX-1728 Pherf - Make tests use mini cluster Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/254024f9 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/254024f9 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/254024f9 Branch: refs/heads/master Commit: 254024f9c97ffa541fbc097a89e2bd2859fa02a2 Parents: 136f7ee Author: cmarcel <[email protected]> Authored: Thu Apr 23 15:26:48 2015 -0700 Committer: cmarcel <[email protected]> Committed: Thu Apr 23 15:26:48 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/pherf/DataIngestIT.java | 88 ++++++++++++++++++++ .../apache/phoenix/pherf/SchemaReaderIT.java | 77 +++++++++++++++++ .../apache/phoenix/pherf/RowCalculatorTest.java | 88 ++++++++++++++++++++ 3 files changed, 253 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java new file mode 100644 index 0000000..b29656d --- /dev/null +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf; + +import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataTypeMapping; +import org.apache.phoenix.pherf.configuration.Scenario; +import org.apache.phoenix.pherf.configuration.XMLConfigParser; +import org.apache.phoenix.pherf.loaddata.DataLoader; +import org.apache.phoenix.pherf.rules.DataValue; +import org.apache.phoenix.pherf.rules.RulesApplier; +import org.apache.phoenix.pherf.schema.SchemaReader; +import org.apache.phoenix.pherf.util.PhoenixUtil; +import org.junit.Test; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class DataIngestIT extends BaseHBaseManagedTimeIT { + protected static PhoenixUtil util = new PhoenixUtil(true); + static final String matcherScenario = ".*scenario/.*test.*xml"; + static final String matcherSchema = ".*datamodel/.*test.*sql"; + + @Test + public void generateData() throws Exception { + util.setZookeeper("localhost"); + SchemaReader reader = new SchemaReader(util, matcherSchema); + XMLConfigParser parser = new XMLConfigParser(matcherScenario); + + // 1. Generate table schema from file + List<Path> resources = new ArrayList<>(reader.getResourceList()); + assertTrue("Could not pull list of schema files.", resources.size() > 0); + assertNotNull("Could not read schema file.", reader.resourceToString(resources.get(0))); + reader.applySchema(); + + // 2. Load the metadata of for the test tables + Scenario scenario = parser.getScenarios().get(0); + List<Column> columnListFromPhoenix = util.getColumnsFromPhoenix(scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), util.getConnection()); + assertTrue("Could not get phoenix columns.", columnListFromPhoenix.size() > 0); + DataLoader loader = new DataLoader(util,parser); + RulesApplier rulesApplier = loader.getRulesApplier(); + List<Map> modelList = rulesApplier.getModelList(); + assertTrue("Could not generate the modelList", modelList.size() > 0); + + for (Column column : columnListFromPhoenix) { + DataValue data = rulesApplier.getDataForRule(scenario, column); + + // We are generating data values so the value should have been specified by this point. + assertTrue("Failed to retrieve data for column type: " + column.getType(), data != null); + + // Test that we still retrieve the GENERAL_CHAR rule even after an override is applied to another CHAR type. + // NEWVAL_STRING Column does not specify an override so we should get the default rule. + if ((column.getType() == DataTypeMapping.VARCHAR) && (column.getName().equals("NEWVAL_STRING"))) { + assertTrue("Failed to retrieve data for column type: ", data.getDistribution() == Integer.MIN_VALUE); + } + } + + // Load up the data. + try { + loader.execute(); + } catch (Exception e) { + fail("Failed to lead data. An exception was thrown: " + e.getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java new file mode 100644 index 0000000..2cb7c13 --- /dev/null +++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf; + +import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.pherf.configuration.Column; +import org.apache.phoenix.pherf.configuration.DataModel; +import org.apache.phoenix.pherf.configuration.Scenario; +import org.apache.phoenix.pherf.configuration.XMLConfigParser; +import org.apache.phoenix.pherf.schema.SchemaReader; +import org.apache.phoenix.pherf.util.PhoenixUtil; +import org.junit.Test; + +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SchemaReaderIT extends BaseHBaseManagedTimeIT { + protected static PhoenixUtil util = new PhoenixUtil(true); + + @Test + public void testSchemaReader() { + // Test for the unit test version of the schema files. + assertApplySchemaTest(); + } + + private void assertApplySchemaTest() { + try { + util.setZookeeper("localhost"); + SchemaReader reader = new SchemaReader(util, ".*datamodel/.*test.*sql"); + + List<Path> resources = new ArrayList<>(reader.getResourceList()); + assertTrue("Could not pull list of schema files.", resources.size() > 0); + assertNotNull("Could not read schema file.", this.getClass().getResourceAsStream( + PherfConstants.RESOURCE_DATAMODEL + "/" + resources.get(0).getFileName().toString())); + assertNotNull("Could not read schema file.", reader.resourceToString(resources.get(0))); + reader.applySchema(); + + Connection connection = null; + URL resourceUrl = getClass().getResource("/scenario/test_scenario.xml"); + assertNotNull("Test data XML file is missing", resourceUrl); + connection = util.getConnection(); + Path resourcePath = Paths.get(resourceUrl.toURI()); + DataModel data = XMLConfigParser.readDataModel(resourcePath); + List<Scenario> scenarioList = data.getScenarios(); + Scenario scenario = scenarioList.get(0); + List<Column> columnList = util.getColumnsFromPhoenix(scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), connection); + assertTrue("Could not retrieve Metadata from Phoenix", columnList.size() > 0); + } catch (Exception e) { + fail("Could not initialize SchemaReader"); + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java ---------------------------------------------------------------------- diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java new file mode 100644 index 0000000..8840289 --- /dev/null +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.pherf; + +import org.apache.phoenix.pherf.util.RowCalculator; +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; + +public class RowCalculatorTest { + /** + * Test rows divide evenly with large rows and small threadpool + * @throws Exception + */ + @Test + public void testRowsEvenDivide() throws Exception { + int threadPoolSize = 10; + int tableRowCount = 100; + assertRowsSum(threadPoolSize, tableRowCount); + } + + /** + * Test rows add up when not divided evenly with large rows and small threadpool + * + * @throws Exception + */ + @Test + public void testRowsNotEvenDivide() throws Exception { + int threadPoolSize = 9; + int tableRowCount = 100; + assertRowsSum(threadPoolSize, tableRowCount); + } + + /** + * Test rows add up when not divided evenly with large threadpool and small rowcount + * + * @throws Exception + */ + @Test + public void testRowsNotEvenDivideSmallRC() throws Exception { + int threadPoolSize = 50; + int tableRowCount = 21; + assertRowsSum(threadPoolSize, tableRowCount); + } + + /** + * Test rows count equal to thread pool + * + * @throws Exception + */ + @Test + public void testRowsEqualToPool() throws Exception { + int threadPoolSize = 50; + int tableRowCount = 50; + assertRowsSum(threadPoolSize, tableRowCount); + } + + private void assertRowsSum(int threadPoolSize, int tableRowCount) { + int sum = 0; + RowCalculator rc = new RowCalculator(threadPoolSize, tableRowCount); + assertEquals("Rows generated did not match expected count! ", threadPoolSize, rc.size()); + + // Sum of all rows should equal expected row count + for (int i = 0; i < threadPoolSize; i++) { + sum += rc.getNext(); + } + assertEquals("Rows did not sum up correctly", tableRowCount, sum); + + // Ensure rows were removed from list + assertEquals(rc.size(), 0); + } +}
