Repository: phoenix Updated Branches: refs/heads/4.x-HBase-0.98 1e14f4664 -> ac62506fb
PHOENIX-2419 Fix flapping Pig tests Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ac62506f Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ac62506f Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ac62506f Branch: refs/heads/4.x-HBase-0.98 Commit: ac62506fbc203a351b5a90c45c674b368ebc8ed5 Parents: 1e14f46 Author: James Taylor <[email protected]> Authored: Sun Nov 15 20:55:18 2015 -0800 Committer: James Taylor <[email protected]> Committed: Sun Nov 15 20:56:08 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/phoenix/pig/BasePigIT.java | 83 ++++++++++++++++++++ .../phoenix/pig/PhoenixHBaseLoaderIT.java | 54 +------------ .../phoenix/pig/PhoenixHBaseStorerIT.java | 59 +------------- .../phoenix/pig/udf/ReserveNSequenceTestIT.java | 62 +++++---------- 4 files changed, 105 insertions(+), 153 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/ac62506f/phoenix-pig/src/it/java/org/apache/phoenix/pig/BasePigIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/BasePigIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/BasePigIT.java new file mode 100644 index 0000000..94ccc25 --- /dev/null +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/BasePigIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.pig; + +import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; +import static org.apache.phoenix.util.TestUtil.LOCALHOST; +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.pig.data.TupleFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; + +import com.google.common.collect.Maps; + +public class BasePigIT extends BaseHBaseManagedTimeIT { + protected TupleFactory tupleFactory; + protected String zkQuorum; + protected Connection conn; + protected Configuration conf; + protected PigServer pigServer; + + @BeforeClass + @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) + public static void doSetup() throws Exception { + Map<String,String> props = Maps.newHashMapWithExpectedSize(3); + props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + // Must update config before starting server + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Before + public void setUp() throws Exception { + conf = getTestClusterConfig(); + conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + pigServer = new PigServer(ExecType.LOCAL, conf); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + conn = DriverManager.getConnection(getUrl(), props); + zkQuorum = LOCALHOST + JDBC_PROTOCOL_SEPARATOR + getZKClientPort(conf); + tupleFactory = TupleFactory.getInstance(); + } + + @After + public void tearDown() throws Exception { + if(conn != null) { + conn.close(); + } + if (pigServer != null) { + pigServer.shutdown(); + } + } + + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/ac62506f/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java index b83ad31..bc614bf 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java @@ -19,54 +19,34 @@ */ package org.apache.phoenix.pig; -import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; -import static org.apache.phoenix.util.TestUtil.LOCALHOST; -import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; -import org.apache.phoenix.end2end.Shadower; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; import org.apache.pig.builtin.mock.Storage; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; /** * * Test class to run all the integration tests against a virtual map reduce cluster. */ -public class PhoenixHBaseLoaderIT extends BaseHBaseManagedTimeIT { +public class PhoenixHBaseLoaderIT extends BasePigIT { private static final Log LOG = LogFactory.getLog(PhoenixHBaseLoaderIT.class); private static final String SCHEMA_NAME = "T"; @@ -75,38 +55,6 @@ public class PhoenixHBaseLoaderIT extends BaseHBaseManagedTimeIT { private static final String TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME); private static final String CASE_SENSITIVE_TABLE_NAME = SchemaUtil.getEscapedArgument("a"); private static final String CASE_SENSITIVE_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME,CASE_SENSITIVE_TABLE_NAME); - private String zkQuorum; - private Connection conn; - private PigServer pigServer; - - @BeforeClass - @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) - public static void doSetup() throws Exception { - Map<String,String> props = Maps.newHashMapWithExpectedSize(3); - props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - // Must update config before starting server - setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); - } - - @Before - public void setUp() throws Exception { - Configuration conf = getTestClusterConfig(); - conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - pigServer = new PigServer(ExecType.LOCAL, conf); - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - conn = DriverManager.getConnection(getUrl(), props); - zkQuorum = LOCALHOST + JDBC_PROTOCOL_SEPARATOR + getZKClientPort(conf); - } - - @After - public void tearDown() throws Exception { - if(conn != null) { - conn.close(); - } - if (pigServer != null) { - pigServer.shutdown(); - } - } /** * Validates the schema returned for a table with Pig data types. http://git-wip-us.apache.org/repos/asf/phoenix/blob/ac62506f/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java index 9c04d68..9e9434c 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java @@ -19,85 +19,28 @@ */ package org.apache.phoenix.pig; -import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; -import static org.apache.phoenix.util.TestUtil.LOCALHOST; -import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; import java.util.Collection; -import java.util.Map; -import java.util.Properties; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; -import org.apache.phoenix.end2end.Shadower; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; -import org.apache.pig.ExecType; -import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS; import org.apache.pig.builtin.mock.Storage; import org.apache.pig.builtin.mock.Storage.Data; import org.apache.pig.data.DataByteArray; import org.apache.pig.data.Tuple; -import org.apache.pig.data.TupleFactory; import org.joda.time.DateTime; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - - -public class PhoenixHBaseStorerIT extends BaseHBaseManagedTimeIT { - private TupleFactory tupleFactory; - private String zkQuorum; - private Connection conn; - private PigServer pigServer; - - @BeforeClass - @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) - public static void doSetup() throws Exception { - Map<String,String> props = Maps.newHashMapWithExpectedSize(3); - props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - // Must update config before starting server - setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); - } - - @Before - public void setUp() throws Exception { - Configuration conf = getTestClusterConfig(); - conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - pigServer = new PigServer(ExecType.LOCAL, conf); - tupleFactory = TupleFactory.getInstance(); - - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - conn = DriverManager.getConnection(getUrl(), props); - zkQuorum = LOCALHOST + JDBC_PROTOCOL_SEPARATOR + getZKClientPort(conf); - } - @After - public void tearDown() throws Exception { - if(conn != null) { - conn.close(); - } - if (pigServer != null) { - pigServer.shutdown(); - } - } +public class PhoenixHBaseStorerIT extends BasePigIT { /** * Basic test - writes data to a Phoenix table and compares the data written * to expected http://git-wip-us.apache.org/repos/asf/phoenix/blob/ac62506f/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java index 16a9520..07be4bc 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/udf/ReserveNSequenceTestIT.java @@ -20,8 +20,6 @@ package org.apache.phoenix.pig.udf; -import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; -import static org.apache.phoenix.util.TestUtil.LOCALHOST; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -32,17 +30,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.pig.BasePigIT; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.pig.data.Tuple; -import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.util.UDFContext; import org.junit.After; -import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -50,37 +43,33 @@ import org.junit.rules.ExpectedException; /** * Test class to run all the Pig Sequence UDF integration tests against a virtual map reduce cluster. */ -public class ReserveNSequenceTestIT extends BaseHBaseManagedTimeIT { +public class ReserveNSequenceTestIT extends BasePigIT { private static final String CREATE_SEQUENCE_SYNTAX = "CREATE SEQUENCE %s START WITH %s INCREMENT BY %s MINVALUE %s MAXVALUE %s CACHE %s"; private static final String SEQUENCE_NAME = "my_schema.my_sequence"; private static final long MAX_VALUE = 10; - private static TupleFactory TF; - private static Connection globalConn; - private static String zkQuorum; - private static Configuration conf; private static UDFContext udfContext; @Rule public ExpectedException thrown = ExpectedException.none(); - @BeforeClass - public static void setUpBeforeClass() throws Exception { - conf = getTestClusterConfig(); - zkQuorum = LOCALHOST + JDBC_PROTOCOL_SEPARATOR + getZKClientPort(getTestClusterConfig()); - conf.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum); - globalConn = DriverManager.getConnection(getUrl()); - // Pig variables - TF = TupleFactory.getInstance(); - } - + @Override @Before - public void setUp() throws SQLException { - createSequence(globalConn); + public void setUp() throws Exception { + super.setUp(); + createSequence(conn); createUdfContext(); } + @Override + @After + public void tearDown() throws Exception { + udfContext.reset(); + dropSequence(conn); + super.tearDown(); + } + @Test public void testReserve() throws Exception { doTest(new UDFTestProperties(1)); @@ -164,7 +153,7 @@ public class ReserveNSequenceTestIT extends BaseHBaseManagedTimeIT { doTest(tenantConn, props); // validate global sequence value is still set to 1 - assertEquals(1L, getNextSequenceValue(globalConn)); + assertEquals(1L, getNextSequenceValue(conn)); } finally { dropSequence(tenantConn); } @@ -177,27 +166,27 @@ public class ReserveNSequenceTestIT extends BaseHBaseManagedTimeIT { */ @Test public void testMultipleTuples() throws Exception { - Tuple tuple = TF.newTuple(2); + Tuple tuple = tupleFactory.newTuple(2); tuple.set(0, 2L); tuple.set(1, SEQUENCE_NAME); - final String tentantId = globalConn.getClientInfo(PhoenixRuntime.TENANT_ID_ATTRIB); + final String tentantId = conn.getClientInfo(PhoenixRuntime.TENANT_ID_ATTRIB); ReserveNSequence udf = new ReserveNSequence(zkQuorum, tentantId); for (int i = 0; i < 2; i++) { udf.exec(tuple); } - long nextValue = getNextSequenceValue(globalConn); + long nextValue = getNextSequenceValue(conn); assertEquals(5L, nextValue); } private void doTest(UDFTestProperties props) throws Exception { - doTest(globalConn, props); + doTest(conn, props); } private void doTest(Connection conn, UDFTestProperties props) throws Exception { setCurrentValue(conn, props.getCurrentValue()); - Tuple tuple = TF.newTuple(3); + Tuple tuple = tupleFactory.newTuple(3); tuple.set(0, props.getNumToReserve()); tuple.set(1, props.getSequenceName()); tuple.set(2, zkQuorum); @@ -254,17 +243,6 @@ public class ReserveNSequenceTestIT extends BaseHBaseManagedTimeIT { conn.commit(); } - @After - public void tearDown() throws Exception { - udfContext.reset(); - dropSequence(globalConn); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - globalConn.close(); - } - /** * Static class to define properties for the test */
