This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix-connectors.git
The following commit(s) were added to refs/heads/master by this push: new 0583b22 PHOENIX-6443 Update Phoenix dependency to 5.1.1 0583b22 is described below commit 0583b225a78a1f3c585735eb2dc46f62d091b33f Author: Istvan Toth <st...@apache.org> AuthorDate: Thu Apr 15 19:59:48 2021 +0200 PHOENIX-6443 Update Phoenix dependency to 5.1.1 previous commit should have been part of this one --- phoenix-flume-base/phoenix5-flume/pom.xml | 2 +- phoenix-flume-base/pom.xml | 9 +++ .../apache/phoenix/flume/CsvEventSerializerIT.java | 27 ++++++++- .../phoenix/flume/JsonEventSerializerIT.java | 29 ++++++++- .../org/apache/phoenix/flume/PhoenixSinkIT.java | 31 ++++++++-- .../phoenix/flume/RegexEventSerializerIT.java | 30 ++++++++-- .../phoenix4-hive-shaded}/pom.xml | 47 ++++++++------- phoenix-hive-base/phoenix4-hive/pom.xml | 20 +++++-- .../phoenix5-hive-shaded}/pom.xml | 52 ++++++++++------ phoenix-hive-base/phoenix5-hive/pom.xml | 18 ++++-- phoenix-hive-base/pom.xml | 7 +-- phoenix-kafka-base/phoenix5-kafka/pom.xml | 1 + phoenix-kafka-base/pom.xml | 21 ++++--- .../apache/phoenix/kafka/PhoenixConsumerIT.java | 25 ++++++-- .../phoenix4-pig-shaded}/pom.xml | 51 ++++++++-------- phoenix-pig-base/phoenix4-pig/pom.xml | 4 -- .../phoenix5-pig-shaded}/pom.xml | 51 +++++++++------- phoenix-pig-base/phoenix5-pig/pom.xml | 6 +- phoenix-pig-base/pom.xml | 15 ++++- .../it/java/org/apache/phoenix/pig/BasePigIT.java | 13 ++-- .../pom.xml | 39 ++++++------ phoenix-spark-base/phoenix4-spark/pom.xml | 4 -- .../phoenix5-spark-shaded}/pom.xml | 39 +++++++----- phoenix-spark-base/phoenix5-spark/pom.xml | 39 +----------- phoenix-spark-base/pom.xml | 20 +++++-- .../phoenix/spark/AbstractPhoenixSparkIT.scala | 20 ++++--- phoenix4-connectors-assembly/pom.xml | 22 +++++-- .../src/build/components/phoenix4-jars.xml | 12 ++-- phoenix5-connectors-assembly/pom.xml | 24 ++++++-- .../src/build/components/phoenix5-jars.xml | 12 ++-- pom.xml | 69 ++++++++++++++++------ 31 files changed, 484 insertions(+), 275 deletions(-) diff --git a/phoenix-flume-base/phoenix5-flume/pom.xml b/phoenix-flume-base/phoenix5-flume/pom.xml index ffc1aa7..eef8e22 100644 --- a/phoenix-flume-base/phoenix5-flume/pom.xml +++ b/phoenix-flume-base/phoenix5-flume/pom.xml @@ -36,6 +36,7 @@ <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> @@ -65,5 +66,4 @@ </plugins> </build> - </project> \ No newline at end of file diff --git a/phoenix-flume-base/pom.xml b/phoenix-flume-base/pom.xml index 0dd114c..e3baa66 100644 --- a/phoenix-flume-base/pom.xml +++ b/phoenix-flume-base/pom.xml @@ -46,6 +46,12 @@ <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> <dependency> <groupId>commons-collections</groupId> @@ -145,6 +151,9 @@ <ignoredUnusedDeclaredDependency> org.apache.hadoop:hadoop-common </ignoredUnusedDeclaredDependency> + <ignoredUnusedDeclaredDependency> + org.apache.phoenix:phoenix-hbase-compat-${hbase.compat.version} + </ignoredUnusedDeclaredDependency> </ignoredUnusedDeclaredDependencies> </configuration> </plugin> diff --git a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/CsvEventSerializerIT.java b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/CsvEventSerializerIT.java index 1f4fff9..1db9d2f 100644 --- a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/CsvEventSerializerIT.java +++ b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/CsvEventSerializerIT.java @@ -41,19 +41,42 @@ import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.apache.flume.lifecycle.LifecycleState; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.flume.serializer.EventSerializers; import org.apache.phoenix.flume.sink.PhoenixSink; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.query.BaseTest; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; -public class CsvEventSerializerIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class CsvEventSerializerIT extends BaseTest { private Context sinkContext; private PhoenixSink sink; + @BeforeClass + public static synchronized void doSetup() throws Exception { + setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); + } + + @AfterClass + public static synchronized void doTeardown() throws Exception { + dropNonSystemTables(); + } + + @After + public void cleanUpAfterTest() throws Exception { + deletePriorMetaData(HConstants.LATEST_TIMESTAMP, getUrl()); + } + @Test public void testWithDefaultDelimiters() throws EventDeliveryException, SQLException { diff --git a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/JsonEventSerializerIT.java b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/JsonEventSerializerIT.java index 2265350..d53a560 100644 --- a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/JsonEventSerializerIT.java +++ b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/JsonEventSerializerIT.java @@ -41,19 +41,42 @@ import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.apache.flume.lifecycle.LifecycleState; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.flume.serializer.EventSerializers; import org.apache.phoenix.flume.sink.PhoenixSink; +import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; - -public class JsonEventSerializerIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class JsonEventSerializerIT extends BaseTest { private Context sinkContext; private PhoenixSink sink; + @BeforeClass + public static synchronized void doSetup() throws Exception { + setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); + } + + @AfterClass + public static synchronized void doTeardown() throws Exception { + dropNonSystemTables(); + } + + @After + public void cleanUpAfterTest() throws Exception { + deletePriorMetaData(HConstants.LATEST_TIMESTAMP, getUrl()); + } + @Test public void testWithOutColumnsMapping() throws EventDeliveryException, SQLException { diff --git a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/PhoenixSinkIT.java b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/PhoenixSinkIT.java index 2e18d53..5c5d884 100644 --- a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/PhoenixSinkIT.java +++ b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/PhoenixSinkIT.java @@ -38,25 +38,46 @@ import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.apache.flume.lifecycle.LifecycleState; import org.apache.flume.sink.DefaultSinkFactory; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compat.CompatUtil; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.flume.serializer.CustomSerializer; import org.apache.phoenix.flume.serializer.EventSerializers; import org.apache.phoenix.flume.sink.NullPhoenixSink; import org.apache.phoenix.flume.sink.PhoenixSink; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.TestUtil; +import org.apache.phoenix.query.BaseTest; import org.junit.Assert; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; - -public class PhoenixSinkIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class PhoenixSinkIT extends BaseTest { private Context sinkContext; private PhoenixSink sink; - - + + @BeforeClass + public static synchronized void doSetup() throws Exception { + setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); + } + + @AfterClass + public static synchronized void doTeardown() throws Exception { + dropNonSystemTables(); + } + + @After + public void cleanUpAfterTest() throws Exception { + deletePriorMetaData(HConstants.LATEST_TIMESTAMP, getUrl()); + } + @Test public void testSinkCreation() { SinkFactory factory = new DefaultSinkFactory (); diff --git a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/RegexEventSerializerIT.java b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/RegexEventSerializerIT.java index 92c85e1..b849f10 100644 --- a/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/RegexEventSerializerIT.java +++ b/phoenix-flume-base/src/it/java/org/apache/phoenix/flume/RegexEventSerializerIT.java @@ -41,19 +41,41 @@ import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.apache.flume.lifecycle.LifecycleState; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.flume.serializer.EventSerializers; import org.apache.phoenix.flume.sink.PhoenixSink; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.query.BaseTest; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; - -public class RegexEventSerializerIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class RegexEventSerializerIT extends BaseTest { private Context sinkContext; private PhoenixSink sink; - + + @BeforeClass + public static synchronized void doSetup() throws Exception { + setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); + } + + @AfterClass + public static synchronized void doTeardown() throws Exception { + dropNonSystemTables(); + } + + @After + public void cleanUpAfterTest() throws Exception { + deletePriorMetaData(HConstants.LATEST_TIMESTAMP, getUrl()); + } + @Test public void testKeyGenerator() throws EventDeliveryException, SQLException { diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-hive-base/phoenix4-hive-shaded/pom.xml similarity index 62% copy from phoenix-kafka-base/phoenix5-kafka/pom.xml copy to phoenix-hive-base/phoenix4-hive-shaded/pom.xml index 3049897..31d24b4 100644 --- a/phoenix-kafka-base/phoenix5-kafka/pom.xml +++ b/phoenix-hive-base/phoenix4-hive-shaded/pom.xml @@ -23,29 +23,30 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> - <artifactId>phoenix-kafka-base</artifactId> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hive-base</artifactId> <version>6.0.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix5-kafka</artifactId> - <name>Phoenix Kafka Connector for Phoenix 5</name> + <artifactId>phoenix4-hive-shaded</artifactId> + <name>Shaded Phoenix Hive Connector for Phoenix 4</name> <properties> - <top.dir>${project.basedir}/..</top.dir> - <phoenix.version>${phoenix-five.version}</phoenix.version> - <hbase.version>${hbase-two.version}</hbase.version> - <hadoop.version>${hadoop-three.version}</hadoop.version> - <jdk.version>1.8</jdk.version> - <phoenix.main.version>5</phoenix.main.version> + <hive.version>${hive2.version}</hive.version> + <hive-storage.version>${hive2-storage.version}</hive-storage.version> + <log4j2.version>2.6.2</log4j2.version> </properties> - <dependencies> <dependency> <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix5-flume</artifactId> + <artifactId>phoenix4-hive</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> </dependency> </dependencies> @@ -53,23 +54,25 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> + <!-- TODO consider not enabling these globally instead --> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> diff --git a/phoenix-hive-base/phoenix4-hive/pom.xml b/phoenix-hive-base/phoenix4-hive/pom.xml index 97f2479..ac5a95a 100644 --- a/phoenix-hive-base/phoenix4-hive/pom.xml +++ b/phoenix-hive-base/phoenix4-hive/pom.xml @@ -42,10 +42,16 @@ <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-connectors-phoenix4-compat</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> <!-- Hive dependencies cannot be specified in the phoenix-hive-common, because of hbase transitive dependency version conflicts --> - </dependency> - <dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-cli</artifactId> <version>${hive.version}</version> @@ -93,6 +99,12 @@ <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> @@ -109,10 +121,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-hive-base/phoenix5-hive-shaded/pom.xml similarity index 60% copy from phoenix-kafka-base/phoenix5-kafka/pom.xml copy to phoenix-hive-base/phoenix5-hive-shaded/pom.xml index 3049897..f649031 100644 --- a/phoenix-kafka-base/phoenix5-kafka/pom.xml +++ b/phoenix-hive-base/phoenix5-hive-shaded/pom.xml @@ -23,55 +23,69 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> - <artifactId>phoenix-kafka-base</artifactId> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hive-base</artifactId> <version>6.0.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix5-kafka</artifactId> - <name>Phoenix Kafka Connector for Phoenix 5</name> + <artifactId>phoenix5-hive-shaded</artifactId> + <name>Shaded Phoenix Hive Connector for Phoenix 5</name> <properties> - <top.dir>${project.basedir}/..</top.dir> + <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> + <zookeeper.version>${hbase-two.zookeeper.version}</zookeeper.version> + <hive.version>${hive3.version}</hive.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> </properties> - <dependencies> <dependency> <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix5-flume</artifactId> + <artifactId>phoenix5-hive</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper-jute</artifactId> + <version>${zookeeper.version}</version> + <scope>provided</scope> </dependency> </dependencies> - + <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> + <!-- TODO consider not enabling these globally instead --> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> - </project> \ No newline at end of file diff --git a/phoenix-hive-base/phoenix5-hive/pom.xml b/phoenix-hive-base/phoenix5-hive/pom.xml index e9bede2..9a3a8cd 100644 --- a/phoenix-hive-base/phoenix5-hive/pom.xml +++ b/phoenix-hive-base/phoenix5-hive/pom.xml @@ -36,7 +36,9 @@ <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> + <zookeeper.version>${hbase-two.zookeeper.version}</zookeeper.version> <hive.version>${hive3.version}</hive.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> @@ -47,6 +49,12 @@ <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-connectors-phoenix5-compat</artifactId> </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> <!-- Hive dependencies cannot be specified in the phoenix-hive-common, because of hbase transitive dependency version conflicts --> <dependency> @@ -141,6 +149,12 @@ <artifactId>hadoop-minicluster</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> <plugins> @@ -156,10 +170,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> </project> \ No newline at end of file diff --git a/phoenix-hive-base/pom.xml b/phoenix-hive-base/pom.xml index 0bcd6c3..f1652dd 100644 --- a/phoenix-hive-base/pom.xml +++ b/phoenix-hive-base/pom.xml @@ -36,7 +36,9 @@ <modules> <module>phoenix4-hive</module> + <module>phoenix4-hive-shaded</module> <module>phoenix5-hive</module> + <module>phoenix5-hive-shaded</module> </modules> <properties> @@ -247,11 +249,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> <scope>provided</scope> diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-kafka-base/phoenix5-kafka/pom.xml index 3049897..8b2323e 100644 --- a/phoenix-kafka-base/phoenix5-kafka/pom.xml +++ b/phoenix-kafka-base/phoenix5-kafka/pom.xml @@ -36,6 +36,7 @@ <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> diff --git a/phoenix-kafka-base/pom.xml b/phoenix-kafka-base/pom.xml index 717cee4..f2be59a 100644 --- a/phoenix-kafka-base/pom.xml +++ b/phoenix-kafka-base/pom.xml @@ -62,6 +62,12 @@ <artifactId>phoenix-core</artifactId> </dependency> <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.11</artifactId> <version>${kafka.version}</version> @@ -96,6 +102,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.phoenix.thirdparty</groupId> + <artifactId>phoenix-shaded-guava</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -134,15 +145,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <configuration> - <!-- AFAICT these are bogus dependency problems --> - <ignoredUnusedDeclaredDependencies> - <ignoredUnusedDeclaredDependency> - org.apache.hbase:hbase-it - </ignoredUnusedDeclaredDependency> - <ignoredUnusedDeclaredDependency> - org.apache.hadoop:hadoop-minicluster - </ignoredUnusedDeclaredDependency> - </ignoredUnusedDeclaredDependencies> + <ignoreNonCompile>true</ignoreNonCompile> </configuration> </plugin> <plugin> diff --git a/phoenix-kafka-base/src/it/java/org/apache/phoenix/kafka/PhoenixConsumerIT.java b/phoenix-kafka-base/src/it/java/org/apache/phoenix/kafka/PhoenixConsumerIT.java index 6de0a47..cb41008 100644 --- a/phoenix-kafka-base/src/it/java/org/apache/phoenix/kafka/PhoenixConsumerIT.java +++ b/phoenix-kafka-base/src/it/java/org/apache/phoenix/kafka/PhoenixConsumerIT.java @@ -27,22 +27,29 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Map; import java.util.Properties; import org.I0Itec.zkclient.ZkClient; import org.apache.flume.Context; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.flume.DefaultKeyGenerator; import org.apache.phoenix.flume.FlumeConstants; import org.apache.phoenix.flume.serializer.EventSerializers; import org.apache.phoenix.kafka.consumer.PhoenixConsumer; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.query.BaseTest; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; - +import org.junit.experimental.categories.Category; import kafka.admin.AdminUtils; @@ -55,7 +62,8 @@ import kafka.utils.ZKStringSerializer$; import kafka.utils.ZkUtils; import kafka.zk.EmbeddedZookeeper; -public class PhoenixConsumerIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class PhoenixConsumerIT extends BaseTest { private static final String ZKHOST = "127.0.0.1"; private static final String BROKERHOST = "127.0.0.1"; private static final String BROKERPORT = "9092"; @@ -66,8 +74,17 @@ public class PhoenixConsumerIT extends BaseHBaseManagedTimeIT { private ZkClient zkClient; private Connection conn; + @BeforeClass + public static void doSetup() throws Exception { + Map<String,String> props = Maps.newHashMapWithExpectedSize(3); + props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + // Must update config before starting server + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + @Before - public void setUp() throws IOException, SQLException { + public void setUp() throws Exception { + setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); // setup Zookeeper zkServer = new EmbeddedZookeeper(); String zkConnect = ZKHOST + ":" + zkServer.port(); diff --git a/phoenix-spark-base/phoenix4-spark/pom.xml b/phoenix-pig-base/phoenix4-pig-shaded/pom.xml similarity index 68% copy from phoenix-spark-base/phoenix4-spark/pom.xml copy to phoenix-pig-base/phoenix4-pig-shaded/pom.xml index 903c75f..4ab1ddb 100644 --- a/phoenix-spark-base/phoenix4-spark/pom.xml +++ b/phoenix-pig-base/phoenix4-pig-shaded/pom.xml @@ -23,49 +23,50 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> - <artifactId>phoenix-spark-base</artifactId> + <artifactId>phoenix-pig-base</artifactId> <groupId>org.apache.phoenix</groupId> <version>6.0.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix4-spark</artifactId> - <name>Phoenix Spark Connector for Phoenix 4</name> - - <dependencies> - <dependency> - <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix-connectors-phoenix4-compat</artifactId> - </dependency> - </dependencies> + <artifactId>phoenix4-pig-shaded</artifactId> + <name>Shaded Phoenix Pig Connector for Phoenix 4</name> <build> <plugins> <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> - </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.scalatest</groupId> - <artifactId>scalatest-maven-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> + <dependencies> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-pig</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> </project> \ No newline at end of file diff --git a/phoenix-pig-base/phoenix4-pig/pom.xml b/phoenix-pig-base/phoenix4-pig/pom.xml index 913f5df..1bbdabe 100644 --- a/phoenix-pig-base/phoenix4-pig/pom.xml +++ b/phoenix-pig-base/phoenix4-pig/pom.xml @@ -42,10 +42,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-pig-base/phoenix5-pig-shaded/pom.xml similarity index 69% copy from phoenix-kafka-base/phoenix5-kafka/pom.xml copy to phoenix-pig-base/phoenix5-pig-shaded/pom.xml index 3049897..675ea27 100644 --- a/phoenix-kafka-base/phoenix5-kafka/pom.xml +++ b/phoenix-pig-base/phoenix5-pig-shaded/pom.xml @@ -23,55 +23,62 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> - <artifactId>phoenix-kafka-base</artifactId> + <artifactId>phoenix-pig-base</artifactId> <groupId>org.apache.phoenix</groupId> <version>6.0.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix5-kafka</artifactId> - <name>Phoenix Kafka Connector for Phoenix 5</name> + <artifactId>phoenix5-pig-shaded</artifactId> + <name>Shaded Phoenix Pig Connector for Phoenix 5</name> <properties> <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> </properties> - - <dependencies> - <dependency> - <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix5-flume</artifactId> - </dependency> - </dependencies> - <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> + <dependencies> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-pig</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + </dependency> + </dependencies> + </project> \ No newline at end of file diff --git a/phoenix-pig-base/phoenix5-pig/pom.xml b/phoenix-pig-base/phoenix5-pig/pom.xml index 5b4b702..874799e 100644 --- a/phoenix-pig-base/phoenix5-pig/pom.xml +++ b/phoenix-pig-base/phoenix5-pig/pom.xml @@ -36,9 +36,11 @@ <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> + </properties> <build> @@ -51,10 +53,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-resources-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> diff --git a/phoenix-pig-base/pom.xml b/phoenix-pig-base/pom.xml index d0aebbe..3e4afc0 100644 --- a/phoenix-pig-base/pom.xml +++ b/phoenix-pig-base/pom.xml @@ -34,7 +34,9 @@ <packaging>pom</packaging> <modules> <module>phoenix4-pig</module> + <module>phoenix4-pig-shaded</module> <module>phoenix5-pig</module> + <module>phoenix5-pig-shaded</module> </modules> <properties> @@ -46,7 +48,13 @@ <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> </dependency> - + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> + <dependency> <groupId>org.apache.pig</groupId> <artifactId>pig</artifactId> @@ -133,6 +141,9 @@ <ignoredUnusedDeclaredDependency> org.apache.hbase:hbase-it </ignoredUnusedDeclaredDependency> + <ignoredUnusedDeclaredDependency> + org.apache.phoenix:phoenix-hbase-compat-${hbase.compat.version} + </ignoredUnusedDeclaredDependency> </ignoredUnusedDeclaredDependencies> </configuration> </plugin> @@ -178,7 +189,7 @@ <goal>shade</goal> </goals> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedArtifactAttached>false</shadedArtifactAttached> <promoteTransitiveDependencies>true</promoteTransitiveDependencies> <shadeTestJar>false</shadeTestJar> <transformers> diff --git a/phoenix-pig-base/src/it/java/org/apache/phoenix/pig/BasePigIT.java b/phoenix-pig-base/src/it/java/org/apache/phoenix/pig/BasePigIT.java index 0e64289..8550fe0 100644 --- a/phoenix-pig-base/src/it/java/org/apache/phoenix/pig/BasePigIT.java +++ b/phoenix-pig-base/src/it/java/org/apache/phoenix/pig/BasePigIT.java @@ -27,8 +27,8 @@ import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; -import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -41,10 +41,12 @@ import org.apache.pig.data.TupleFactory; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; -public class BasePigIT extends BaseHBaseManagedTimeIT { +@Category(NeedsOwnMiniClusterTest.class) +public class BasePigIT extends BaseTest { protected TupleFactory tupleFactory; protected String zkQuorum; protected Connection conn; @@ -52,7 +54,6 @@ public class BasePigIT extends BaseHBaseManagedTimeIT { protected PigServer pigServer; @BeforeClass - @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) public static void doSetup() throws Exception { Map<String,String> props = Maps.newHashMapWithExpectedSize(3); props.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); @@ -83,5 +84,9 @@ public class BasePigIT extends BaseHBaseManagedTimeIT { } } + protected static Configuration getTestClusterConfig() { + // don't want callers to modify config. + return new Configuration(config); + } } diff --git a/phoenix-spark-base/phoenix4-spark/pom.xml b/phoenix-spark-base/phoenix4-spark-shaded/pom.xml similarity index 68% copy from phoenix-spark-base/phoenix4-spark/pom.xml copy to phoenix-spark-base/phoenix4-spark-shaded/pom.xml index 903c75f..0d8561a 100644 --- a/phoenix-spark-base/phoenix4-spark/pom.xml +++ b/phoenix-spark-base/phoenix4-spark-shaded/pom.xml @@ -29,41 +29,44 @@ </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix4-spark</artifactId> - <name>Phoenix Spark Connector for Phoenix 4</name> + <artifactId>phoenix4-spark-shaded</artifactId> + <name>Shaded Phoenix Spark Connector for Phoenix 4</name> <dependencies> <dependency> <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix-connectors-phoenix4-compat</artifactId> + <artifactId>phoenix4-spark</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> </dependency> </dependencies> <build> <plugins> <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> - </plugin> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.scalatest</groupId> - <artifactId>scalatest-maven-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> + <!-- TODO consider not enabling these globally instead --> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> diff --git a/phoenix-spark-base/phoenix4-spark/pom.xml b/phoenix-spark-base/phoenix4-spark/pom.xml index 903c75f..53bbfde 100644 --- a/phoenix-spark-base/phoenix4-spark/pom.xml +++ b/phoenix-spark-base/phoenix4-spark/pom.xml @@ -61,10 +61,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> diff --git a/phoenix-kafka-base/phoenix5-kafka/pom.xml b/phoenix-spark-base/phoenix5-spark-shaded/pom.xml similarity index 68% copy from phoenix-kafka-base/phoenix5-kafka/pom.xml copy to phoenix-spark-base/phoenix5-spark-shaded/pom.xml index 3049897..a565823 100644 --- a/phoenix-kafka-base/phoenix5-kafka/pom.xml +++ b/phoenix-spark-base/phoenix5-spark-shaded/pom.xml @@ -23,29 +23,34 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <parent> - <artifactId>phoenix-kafka-base</artifactId> + <artifactId>phoenix-spark-base</artifactId> <groupId>org.apache.phoenix</groupId> <version>6.0.0-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>phoenix5-kafka</artifactId> - <name>Phoenix Kafka Connector for Phoenix 5</name> + <artifactId>phoenix5-spark-shaded</artifactId> + <name>Shaded Phoenix Spark Connector for Phoenix 5</name> <properties> <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> <phoenix.main.version>5</phoenix.main.version> </properties> - <dependencies> <dependency> <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix5-flume</artifactId> + <artifactId>phoenix4-spark</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> </dependency> </dependencies> @@ -53,23 +58,25 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> </plugin> + <!-- TODO consider not enabling these globally instead --> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-site-plugin</artifactId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>default-compile</id> + <phase>none</phase> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> diff --git a/phoenix-spark-base/phoenix5-spark/pom.xml b/phoenix-spark-base/phoenix5-spark/pom.xml index 3a9105f..f33ff88 100644 --- a/phoenix-spark-base/phoenix5-spark/pom.xml +++ b/phoenix-spark-base/phoenix5-spark/pom.xml @@ -36,9 +36,9 @@ <top.dir>${project.basedir}/..</top.dir> <phoenix.version>${phoenix-five.version}</phoenix.version> <hbase.version>${hbase-two.version}</hbase.version> + <hbase.compat.version>${hbase-two.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-three.version}</hadoop.version> <jdk.version>1.8</jdk.version> - <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version> <phoenix.main.version>5</phoenix.main.version> </properties> @@ -47,41 +47,8 @@ <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-connectors-phoenix5-compat</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs-client</artifactId> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </exclusion> - </exclusions> - <scope>provided</scope> - </dependency> </dependencies> - <dependencyManagement> - <dependencies> - <!-- Guava is excluded because of SPARK-6149. The Guava version referenced - in this module is 15.0, which causes runtime incompatibility issues. --> - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-scala_${scala.binary.version}</artifactId> - <version>${fasterxml.jackson.version}</version> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </dependencyManagement> - <build> <plugins> <plugin> @@ -104,10 +71,6 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> </plugins> </build> diff --git a/phoenix-spark-base/pom.xml b/phoenix-spark-base/pom.xml index bff52fb..90c457d 100644 --- a/phoenix-spark-base/pom.xml +++ b/phoenix-spark-base/pom.xml @@ -40,15 +40,14 @@ <packaging>pom</packaging> <modules> <module>phoenix4-spark</module> + <module>phoenix4-spark-shaded</module> <module>phoenix5-spark</module> + <module>phoenix5-spark-shaded</module> </modules> <dependencies> - <dependency> - <groupId>org.apache.phoenix</groupId> - <artifactId>phoenix-core</artifactId> - </dependency> + <!-- Spark dependencies first to avoid jackson compatibility problems --> <!-- Mark Spark / Scala as provided --> <dependency> <groupId>org.scala-lang</groupId> @@ -92,6 +91,17 @@ <version>${spark.version}</version> <scope>provided</scope> </dependency> + + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <scope>runtime</scope> + <optional>true</optional> + </dependency> <!-- Test dependencies --> <dependency> @@ -619,7 +629,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedArtifactAttached>false</shadedArtifactAttached> <promoteTransitiveDependencies>true</promoteTransitiveDependencies> <filters> <filter> diff --git a/phoenix-spark-base/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala b/phoenix-spark-base/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala index a9c2070..f4b21fd 100644 --- a/phoenix-spark-base/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala +++ b/phoenix-spark-base/src/it/scala/org/apache/phoenix/spark/AbstractPhoenixSparkIT.scala @@ -16,29 +16,35 @@ package org.apache.phoenix.spark import java.sql.{Connection, DriverManager} import java.util.Properties -import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.query.BaseTest import org.apache.phoenix.util.PhoenixRuntime +import org.apache.phoenix.util.ReadOnlyProps; import org.apache.spark.sql.{SQLContext, SparkSession} import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite, Matchers} -// Helper object to access the protected abstract static methods hidden in BaseHBaseManagedTimeIT -object PhoenixSparkITHelper extends BaseHBaseManagedTimeIT { - def getTestClusterConfig = BaseHBaseManagedTimeIT.getTestClusterConfig +// Helper object to access the protected abstract static methods hidden in BaseTest +object PhoenixSparkITHelper extends BaseTest { + def getTestClusterConfig = new Configuration(BaseTest.config); def doSetup = { // The @ClassRule doesn't seem to be getting picked up, force creation here before setup BaseTest.tmpFolder.create() - BaseHBaseManagedTimeIT.doSetup() + BaseTest.setUpTestDriver(ReadOnlyProps.EMPTY_PROPS); } def doTeardown = { - BaseHBaseManagedTimeIT.doTeardown() + BaseTest.dropNonSystemTables(); BaseTest.tmpFolder.delete() } + def cleanUpAfterTest = { + BaseTest.deletePriorMetaData(HConstants.LATEST_TIMESTAMP, getUrl); + } + def getUrl = BaseTest.getUrl } @@ -111,7 +117,7 @@ class AbstractPhoenixSparkIT extends FunSuite with Matchers with BeforeAndAfter override def afterAll() { conn.close() spark.stop() - PhoenixSparkITHelper.cleanUpAfterTest() + PhoenixSparkITHelper.cleanUpAfterTest PhoenixSparkITHelper.doTeardown } } diff --git a/phoenix4-connectors-assembly/pom.xml b/phoenix4-connectors-assembly/pom.xml index fd05be6..c7ca9cf 100644 --- a/phoenix4-connectors-assembly/pom.xml +++ b/phoenix4-connectors-assembly/pom.xml @@ -43,6 +43,10 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-hive-shaded</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix4-kafka</artifactId> </dependency> <dependency> @@ -51,8 +55,16 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-pig-shaded</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix4-spark</artifactId> </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-spark-shaded</artifactId> + </dependency> </dependencies> <build> @@ -89,11 +101,11 @@ </goals> <configuration> <executable>ln</executable> - <workingDirectory>${project.basedir}/../phoenix-hive-base/phoenix4-hive/target</workingDirectory> + <workingDirectory>${project.basedir}/../phoenix-hive-base/phoenix4-hive-shaded/target</workingDirectory> <arguments> <argument>-fnsv</argument> <argument> - phoenix4-hive-${project.version}-shaded.jar + phoenix4-hive-shaded-${project.version}.jar </argument> <argument> phoenix4-hive-shaded.jar @@ -109,11 +121,11 @@ </goals> <configuration> <executable>ln</executable> - <workingDirectory>${project.basedir}/../phoenix-pig-base/phoenix4-pig/target</workingDirectory> + <workingDirectory>${project.basedir}/../phoenix-pig-base/phoenix4-pig-shaded/target</workingDirectory> <arguments> <argument>-fnsv</argument> <argument> - phoenix4-pig-${project.version}-shaded.jar + phoenix4-pig-shaded-${project.version}.jar </argument> <argument> phoenix4-pig-shaded.jar @@ -133,7 +145,7 @@ <arguments> <argument>-fnsv</argument> <argument> - phoenix4-spark-${project.version}-shaded.jar + phoenix4-spark-shaded-${project.version}.jar </argument> <argument> phoenix4-spark-shaded.jar diff --git a/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml b/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml index 643e321..b29b380 100644 --- a/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml +++ b/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml @@ -40,26 +40,26 @@ </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-pig-base/phoenix4-pig/target</directory> + <directory>${project.basedir}/../phoenix-pig-base/phoenix4-pig-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix4-pig-${project.version}-shaded.jar</include> + <include>phoenix4-pig-shaded-${project.version}.jar</include> <include>phoenix4-pig-shaded.jar</include> </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-spark-base/phoenix4-spark/target</directory> + <directory>${project.basedir}/../phoenix-spark-base/phoenix4-spark-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix4-spark-${project.version}-shaded.jar</include> + <include>phoenix4-spark-shaded-${project.version}.jar</include> <include>phoenix4-spark-shaded.jar</include> </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-hive-base/phoenix4-hive/target</directory> + <directory>${project.basedir}/../phoenix-hive-base/phoenix4-hive-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix4-hive-${project.version}-shaded.jar</include> + <include>phoenix4-hive-shaded-${project.version}-shaded.jar</include> <include>phoenix4-hive-shaded.jar</include> </includes> </fileSet> diff --git a/phoenix5-connectors-assembly/pom.xml b/phoenix5-connectors-assembly/pom.xml index e8c9ef0..2dfa232 100644 --- a/phoenix5-connectors-assembly/pom.xml +++ b/phoenix5-connectors-assembly/pom.xml @@ -50,6 +50,10 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-hive-shaded</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix5-kafka</artifactId> </dependency> <dependency> @@ -58,8 +62,16 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-pig-shaded</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix5-spark</artifactId> </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-spark-shaded</artifactId> + </dependency> </dependencies> <build> @@ -96,11 +108,11 @@ </goals> <configuration> <executable>ln</executable> - <workingDirectory>${project.basedir}/../phoenix-hive-base/phoenix5-hive/target</workingDirectory> + <workingDirectory>${project.basedir}/../phoenix-hive-base/phoenix5-hive-shaded/target</workingDirectory> <arguments> <argument>-fnsv</argument> <argument> - phoenix5-hive-${project.version}-shaded.jar + phoenix5-hive-shaded-${project.version}.jar </argument> <argument> phoenix5-hive-shaded.jar @@ -116,11 +128,11 @@ </goals> <configuration> <executable>ln</executable> - <workingDirectory>${project.basedir}/../phoenix-pig-base/phoenix5-pig/target</workingDirectory> + <workingDirectory>${project.basedir}/../phoenix-pig-base/phoenix5-pig-shaded/target</workingDirectory> <arguments> <argument>-fnsv</argument> <argument> - phoenix5-pig-${project.version}-shaded.jar + phoenix5-pig-shaded-${project.version}.jar </argument> <argument> phoenix5-pig-shaded.jar @@ -136,11 +148,11 @@ </goals> <configuration> <executable>ln</executable> - <workingDirectory>${project.basedir}/../phoenix-spark-base/phoenix5-spark/target</workingDirectory> + <workingDirectory>${project.basedir}/../phoenix-spark-base/phoenix5-spark-shaded/target</workingDirectory> <arguments> <argument>-fnsv</argument> <argument> - phoenix5-spark-${project.version}-shaded.jar + phoenix5-spark-shaded-${project.version}.jar </argument> <argument> phoenix5-spark-shaded.jar diff --git a/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml b/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml index b0dbc5c..ee066bc 100644 --- a/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml +++ b/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml @@ -40,26 +40,26 @@ </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-pig-base/phoenix5-pig/target</directory> + <directory>${project.basedir}/../phoenix-pig-base/phoenix5-pig-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix5-pig-${project.version}-shaded.jar</include> + <include>phoenix5-pig-shaded-${project.version}.jar</include> <include>phoenix5-pig-shaded.jar</include> </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-spark-base/phoenix5-spark/target</directory> + <directory>${project.basedir}/../phoenix-spark-base/phoenix5-spark-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix5-spark-${project.version}-shaded.jar</include> + <include>phoenix5-spark-shaded-${project.version}.jar</include> <include>phoenix5-spark-shaded.jar</include> </includes> </fileSet> <fileSet> - <directory>${project.basedir}/../phoenix-hive-base/phoenix5-hive/target</directory> + <directory>${project.basedir}/../phoenix-hive-base/phoenix5-hive-shaded/target</directory> <outputDirectory>/</outputDirectory> <includes> - <include>phoenix5-hive-${project.version}-shaded.jar</include> + <include>phoenix5-hive-shaded-${project.version}.jar</include> <include>phoenix5-hive-shaded.jar</include> </includes> </fileSet> diff --git a/pom.xml b/pom.xml index 25ffaf9..deb7d9c 100644 --- a/pom.xml +++ b/pom.xml @@ -56,19 +56,26 @@ <properties> <!-- Phoenix Version --> - <phoenix-four.version>4.15.0-HBase-1.4</phoenix-four.version> - <phoenix-five.version>5.1.0-SNAPSHOT</phoenix-five.version> - <phoenix.thirdparty.version>1.0.0</phoenix.thirdparty.version> + <phoenix-four.version>4.16.0</phoenix-four.version> + <phoenix-five.version>5.1.1</phoenix-five.version> + <phoenix.thirdparty.version>1.1.0</phoenix.thirdparty.version> <!-- Hadoop Versions --> - <hbase-one.version>1.4.0</hbase-one.version> + <hbase-one.version>1.4.10</hbase-one.version> + <hbase-one.compat.version>1.4.0</hbase-one.compat.version> + <hbase-one.zookeeper.version>3.4.12</hbase-one.zookeeper.version> <hbase-two.version>2.1.10</hbase-two.version> + <hbase-two.compat.version>2.1.6</hbase-two.compat.version> + <hbase-two.zookeeper.version>3.5.9</hbase-two.zookeeper.version> <hbase-thirdparty-version>2.1.0</hbase-thirdparty-version> <hadoop-two.version>2.7.5</hadoop-two.version> <hadoop-three.version>3.0.3</hadoop-three.version> <phoenix.version>${phoenix-four.version}</phoenix.version> <hbase.version>${hbase-one.version}</hbase.version> + <hbase.compat.version>${hbase-one.compat.version}</hbase.compat.version> <hadoop.version>${hadoop-two.version}</hadoop.version> + <zookeeper.version>${hbase-one.zookeeper.version}</zookeeper.version> + <!-- General Properties --> <test.output.tofile>true</test.output.tofile> @@ -106,8 +113,7 @@ <com-101tek-zkclient.version>0.7</com-101tek-zkclient.version> <!-- For hive --> <commons-io.version>2.4</commons-io.version> - <zookeeper.version>3.4.12</zookeeper.version> - + <!-- Plugin versions --> <maven-assembly-plugin-version>3.1.1</maven-assembly-plugin-version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> @@ -298,11 +304,11 @@ </goals> </execution> <execution> - <id>HBaseManagedTimeTests</id> + <id>NeedsOwnMiniClusterTest</id> <configuration> - <reuseForks>true</reuseForks> + <reuseForks>false</reuseForks> <argLine>-enableassertions -Xmx3000m -XX:MaxPermSize=256m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine> - <groups>org.apache.phoenix.end2end.HBaseManagedTimeTest</groups> + <groups>org.apache.phoenix.end2end.NeedsOwnMiniClusterTest</groups> </configuration> <goals> <goal>integration-test</goal> @@ -463,6 +469,11 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-${hbase.compat.version}</artifactId> + <version>${phoenix.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> <version>${phoenix.version}</version> <classifier>tests</classifier> @@ -490,16 +501,31 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-pig-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix4-spark</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-spark-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix4-hive</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix4-hive-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix5-flume</artifactId> <version>${project.version}</version> </dependency> @@ -515,16 +541,31 @@ </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-pig-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix5-spark</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-spark-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix5-hive</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix5-hive-shaded</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-connectors-phoenix4-compat</artifactId> <version>${project.version}</version> </dependency> @@ -775,16 +816,6 @@ <!-- General Dependencies --> <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>${zookeeper.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper-jute</artifactId> - <version>${zookeeper.version}</version> - </dependency> - <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> <version>2.5.0</version>