Repository: falcon Updated Branches: refs/heads/master d2fd49e5c -> 27e872240
FALCON-417 Upgrade hive and hcatalog to latest stable version. Contributed by Peeyush Bishnoi Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/27e87224 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/27e87224 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/27e87224 Branch: refs/heads/master Commit: 27e872240d118f2c55f440b653dd5ceb9de98466 Parents: d2fd49e Author: Srikanth Sundarrajan <[email protected]> Authored: Sat Jan 10 23:24:03 2015 +0530 Committer: Srikanth Sundarrajan <[email protected]> Committed: Sat Jan 10 23:24:03 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 6 ++++++ addons/designer/pom.xml | 7 +++---- common/pom.xml | 6 +++--- .../apache/falcon/catalog/CatalogPartition.java | 2 +- .../apache/falcon/catalog/HiveCatalogService.java | 4 ++-- pom.xml | 18 +++++++++++++++--- test-tools/hadoop-webapp/pom.xml | 4 ++-- webapp/pom.xml | 4 ++-- .../falcon/catalog/HiveCatalogServiceIT.java | 12 ++++++------ .../org/apache/falcon/late/LateDataHandlerIT.java | 4 ++-- .../lifecycle/TableStorageFeedEvictorIT.java | 8 ++++---- .../lifecycle/TableStorageFeedReplicationIT.java | 2 +- .../falcon/process/TableStorageProcessIT.java | 2 +- .../org/apache/falcon/resource/TestContext.java | 2 +- .../org/apache/falcon/util/HiveTestUtils.java | 10 +++++----- 15 files changed, 54 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8781a8e..24967a2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,12 @@ Trunk (Unreleased) NEW FEATURES IMPROVEMENTS + FALCON-417 Upgrade Hive and HCatalog to latest stable version. (Peeyush Bishnoi + via Srikanth Sundarrajan) + + FALCON-896 Upgrade oozie to 4.1.0 and make it the default for falcon. (Peeyush + Bishnoi via Srikanth Sundarrajan) + FALCON-749 Improve error printed in falcon CLI. (Suresh Srinivas via Shwetha GS) FALCON-977 Drop incubator/incubating from falcon codebase. (Shwetha GS http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/addons/designer/pom.xml ---------------------------------------------------------------------- diff --git a/addons/designer/pom.xml b/addons/designer/pom.xml index aee0582..26110f3 100644 --- a/addons/designer/pom.xml +++ b/addons/designer/pom.xml @@ -97,8 +97,7 @@ <include.prism>true</include.prism> <slf4j.version>1.7.5</slf4j.version> - <hive.version>0.11.0</hive.version> - <hcatalog.version>0.11.0</hcatalog.version> + <hive.version>0.13.1</hive.version> <jetty.version>6.1.26</jetty.version> <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo> <skipCheck>false</skipCheck> @@ -364,8 +363,8 @@ </dependency> <dependency> - <groupId>org.apache.hcatalog</groupId> - <artifactId>webhcat-java-client</artifactId> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-webhcat-java-client</artifactId> <version>${hcatalog.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/common/pom.xml ---------------------------------------------------------------------- diff --git a/common/pom.xml b/common/pom.xml index 681b22d..d5629b2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -154,12 +154,12 @@ <dependency> <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> + <artifactId>hive-common</artifactId> </dependency> <dependency> - <groupId>org.apache.hcatalog</groupId> - <artifactId>webhcat-java-client</artifactId> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-webhcat-java-client</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java index 032ae38..9e35782 100644 --- a/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java +++ b/common/src/main/java/org/apache/falcon/catalog/CatalogPartition.java @@ -21,7 +21,7 @@ package org.apache.falcon.catalog; import java.util.List; /** - * The CatalogPartition is a wrapper around org.apache.hcatalog.api.HCatPartition. + * The CatalogPartition is a wrapper around org.apache.hive.hcatalog.api.HCatPartition. */ public class CatalogPartition { http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java index f59b83b..25a4a46 100644 --- a/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java +++ b/common/src/main/java/org/apache/falcon/catalog/HiveCatalogService.java @@ -35,8 +35,8 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.io.Text; -import org.apache.hcatalog.api.HCatClient; -import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; +import org.apache.hive.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cd1a8fb..fea00bf 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ <oozie.buildversion>${oozie.version}-falcon</oozie.buildversion> <oozie.forcebuild>false</oozie.forcebuild> <activemq.version>5.4.3</activemq.version> - <hive.version>0.11.0</hive.version> + <hive.version>0.13.1</hive.version> <jetty.version>6.1.26</jetty.version> <jersey.version>1.9</jersey.version> <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo> @@ -877,10 +877,22 @@ </exclusions> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- this is needed for embedded oozie --> <dependency> - <groupId>org.apache.hcatalog</groupId> - <artifactId>webhcat-java-client</artifactId> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-webhcat-java-client</artifactId> <version>${hive.version}</version> <exclusions> <exclusion> http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/test-tools/hadoop-webapp/pom.xml ---------------------------------------------------------------------- diff --git a/test-tools/hadoop-webapp/pom.xml b/test-tools/hadoop-webapp/pom.xml index 71df231..cc246aa 100644 --- a/test-tools/hadoop-webapp/pom.xml +++ b/test-tools/hadoop-webapp/pom.xml @@ -192,8 +192,8 @@ <!-- Oozie dependencies --> <dependency> - <groupId>org.apache.hcatalog</groupId> - <artifactId>webhcat-java-client</artifactId> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-webhcat-java-client</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 7e89125..556b171 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -195,8 +195,8 @@ </dependency> <dependency> - <groupId>org.apache.hcatalog</groupId> - <artifactId>webhcat-java-client</artifactId> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-webhcat-java-client</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java index 6fd23a0..71616e9 100644 --- a/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java +++ b/webapp/src/test/java/org/apache/falcon/catalog/HiveCatalogServiceIT.java @@ -22,12 +22,12 @@ import org.apache.falcon.FalconException; import org.apache.falcon.resource.TestContext; import org.apache.falcon.security.CurrentUser; import org.apache.hadoop.conf.Configuration; -import org.apache.hcatalog.api.HCatAddPartitionDesc; -import org.apache.hcatalog.api.HCatClient; -import org.apache.hcatalog.api.HCatCreateDBDesc; -import org.apache.hcatalog.api.HCatCreateTableDesc; -import org.apache.hcatalog.api.HCatPartition; -import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; +import org.apache.hive.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.api.HCatCreateDBDesc; +import org.apache.hive.hcatalog.api.HCatCreateTableDesc; +import org.apache.hive.hcatalog.api.HCatPartition; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/late/LateDataHandlerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/late/LateDataHandlerIT.java b/webapp/src/test/java/org/apache/falcon/late/LateDataHandlerIT.java index 86da770..ed85db0 100644 --- a/webapp/src/test/java/org/apache/falcon/late/LateDataHandlerIT.java +++ b/webapp/src/test/java/org/apache/falcon/late/LateDataHandlerIT.java @@ -31,8 +31,8 @@ import org.apache.falcon.util.HiveTestUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hcatalog.api.HCatAddPartitionDesc; -import org.apache.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; +import org.apache.hive.hcatalog.api.HCatClient; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedEvictorIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedEvictorIT.java b/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedEvictorIT.java index ba6698c..6982b65 100644 --- a/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedEvictorIT.java +++ b/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedEvictorIT.java @@ -29,10 +29,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; -import org.apache.hcatalog.api.HCatAddPartitionDesc; -import org.apache.hcatalog.api.HCatClient; -import org.apache.hcatalog.api.HCatPartition; -import org.apache.hcatalog.common.HCatException; +import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; +import org.apache.hive.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.api.HCatPartition; +import org.apache.hive.hcatalog.common.HCatException; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedReplicationIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedReplicationIT.java b/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedReplicationIT.java index 181ed05..55610b7 100644 --- a/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedReplicationIT.java +++ b/webapp/src/test/java/org/apache/falcon/lifecycle/TableStorageFeedReplicationIT.java @@ -30,7 +30,7 @@ import org.apache.falcon.util.OozieTestUtils; import org.apache.falcon.util.StartupProperties; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hcatalog.api.HCatPartition; +import org.apache.hive.hcatalog.api.HCatPartition; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowJob; import org.testng.Assert; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java b/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java index b27b692..9cb6850 100644 --- a/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java +++ b/webapp/src/test/java/org/apache/falcon/process/TableStorageProcessIT.java @@ -30,7 +30,7 @@ import org.apache.falcon.util.OozieTestUtils; import org.apache.falcon.util.StartupProperties; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hcatalog.api.HCatPartition; +import org.apache.hive.hcatalog.api.HCatPartition; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowJob; import org.testng.Assert; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/resource/TestContext.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java index 23df745..593079a 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java +++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java @@ -50,7 +50,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.api.HCatClient; import org.testng.Assert; import javax.net.ssl.HostnameVerifier; http://git-wip-us.apache.org/repos/asf/falcon/blob/27e87224/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java b/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java index 9fd1a9d..19274b9 100644 --- a/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java +++ b/webapp/src/test/java/org/apache/falcon/util/HiveTestUtils.java @@ -23,11 +23,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hcatalog.api.HCatClient; -import org.apache.hcatalog.api.HCatCreateDBDesc; -import org.apache.hcatalog.api.HCatCreateTableDesc; -import org.apache.hcatalog.api.HCatPartition; -import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.api.HCatClient; +import org.apache.hive.hcatalog.api.HCatCreateDBDesc; +import org.apache.hive.hcatalog.api.HCatCreateTableDesc; +import org.apache.hive.hcatalog.api.HCatPartition; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import java.util.ArrayList; import java.util.HashMap;
