Repository: phoenix Updated Branches: refs/heads/5.x-HBase-2.0 309ef0b1a -> 8e5b4131b
PHOENIX-4405 Switch to using Hadoop3 Includes a temporary fix to work around a jetty dependency conflict between Hadoop and HBase Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8e5b4131 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8e5b4131 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8e5b4131 Branch: refs/heads/5.x-HBase-2.0 Commit: 8e5b4131b4b9bbbf9aab43bf304504e8238027f0 Parents: 309ef0b Author: Josh Elser <els...@apache.org> Authored: Mon Nov 27 15:25:42 2017 -0500 Committer: Josh Elser <els...@apache.org> Committed: Mon Nov 27 15:55:40 2017 -0500 ---------------------------------------------------------------------- phoenix-core/pom.xml | 21 ++++++++ .../phoenix/trace/PhoenixMetricsSink.java | 2 +- .../recovery/TestPerRegionIndexWriteCache.java | 31 ++++++++++-- .../org/apache/phoenix/metrics/LoggingSink.java | 2 +- phoenix-spark/pom.xml | 8 +-- phoenix-tracing-webapp/pom.xml | 16 ++++-- pom.xml | 52 ++++++++++++++------ 7 files changed, 104 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml index 3124d9b..0d5affe 100644 --- a/phoenix-core/pom.xml +++ b/phoenix-core/pom.xml @@ -418,6 +418,27 @@ </exclusion> </exclusions> </dependency> + <!-- TODO remove after HBASE-19256 --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java index fea6d61..65071f0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java @@ -34,7 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseConfiguration; http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java index 2bfbcbf..c51ac99 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java @@ -43,11 +43,16 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.phoenix.hbase.index.table.HTableInterfaceReference; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -60,6 +65,8 @@ public class TestPerRegionIndexWriteCache { private static final byte[] qual = Bytes.toBytes("qual"); private static final byte[] val = Bytes.toBytes("val"); + private static MiniDFSCluster miniDfs = null; + Put p = new Put(row); Put p2 = new Put(Bytes.toBytes("other row")); { @@ -71,18 +78,34 @@ public class TestPerRegionIndexWriteCache { HRegion r2; // FIXME: Uses private type WAL wal; + @Rule + public TestName testName = new TestName(); + + @BeforeClass + public static void startDfs() throws Exception { + miniDfs = TEST_UTIL.startMiniDFSCluster(1); + } + + @AfterClass + public static void stopDfs() throws Exception { + if (miniDfs != null) { + miniDfs.shutdown(); + miniDfs = null; + } + } + @SuppressWarnings("deprecation") -@Before + @Before public void setUp() throws Exception { - Path hbaseRootDir = TEST_UTIL.getDataTestDir(); + Path hbaseRootDir = new Path(getClass().getSimpleName() + "_" + testName.getMethodName()); TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString()); - FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration()); + FileSystem newFS = miniDfs.getFileSystem(); RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).setSplit(false).build(); Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName); Random rn = new Random(); tableName = TableName.valueOf("TestPerRegion" + rn.nextInt()); - WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache"); + WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, getClass().getSimpleName()); wal = walFactory.getWAL(Bytes.toBytes("logs"), null); TableDescriptor htd = TableDescriptorBuilder http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java index 2cea684..1af01ab 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java @@ -17,7 +17,7 @@ */ package org.apache.phoenix.metrics; -import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics2.AbstractMetric; http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-spark/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml index f3bd9c2..d6ad4d5 100644 --- a/phoenix-spark/pom.xml +++ b/phoenix-spark/pom.xml @@ -102,7 +102,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>log4j</groupId> @@ -134,7 +134,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>log4j</groupId> @@ -166,7 +166,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <type>test-jar</type> <scope>test</scope> <exclusions> @@ -200,7 +200,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <type>test-jar</type> <scope>test</scope> <exclusions> http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-tracing-webapp/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-tracing-webapp/pom.xml b/phoenix-tracing-webapp/pom.xml index 6cc9960..1cb047f 100755 --- a/phoenix-tracing-webapp/pom.xml +++ b/phoenix-tracing-webapp/pom.xml @@ -41,15 +41,23 @@ <dependencies> <dependency> <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> - <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)--> - <version>${jettyVersion}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-webapp</artifactId> - <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)--> - <version>${jettyVersion}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index bef97db..ac5cd08 100644 --- a/pom.xml +++ b/pom.xml @@ -67,12 +67,11 @@ <!-- Hadoop Versions --> <hbase.version>2.0.0-beta-1-SNAPSHOT</hbase.version> - <hadoop-two.version>2.7.1</hadoop-two.version> + <hadoop.version>3.0.0-beta1</hadoop.version> <!-- Dependency versions --> <commons-cli.version>1.2</commons-cli.version> <hive.version>1.2.1</hive.version> - <hadoop.version>2.7.1</hadoop.version> <pig.version>0.13.0</pig.version> <jackson.version>1.9.2</jackson.version> <antlr.version>3.5.2</antlr.version> @@ -99,7 +98,7 @@ <jodatime.version>1.6</jodatime.version> <joni.version>2.1.2</joni.version> <avatica.version>1.10.0</avatica.version> - <jettyVersion>8.1.7.v20120910</jettyVersion> + <jetty.version>9.3.19.v20170502</jetty.version> <tephra.version>0.13.0-incubating</tephra.version> <spark.version>2.0.2</spark.version> <scala.version>2.11.8</scala.version> @@ -655,7 +654,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>org.xerial.snappy</groupId> @@ -666,17 +665,17 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-annotations</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <optional>true</optional> <scope>test</scope> </dependency> @@ -685,39 +684,39 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-common</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-jobclient</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> <type>test-jar</type> <!-- this does not work which is typical for maven.--> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minikdc</artifactId> - <version>${hadoop-two.version}</version> + <version>${hadoop.version}</version> </dependency> <!-- General Dependencies --> @@ -945,6 +944,31 @@ <artifactId>i18n-util</artifactId> <version>${i18n-util.version}</version> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-security</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <version>${jetty.version}</version> + </dependency> </dependencies> </dependencyManagement>