Updated Branches: refs/heads/trunk e65d9c798 -> a9089430e
SQOOP-1172: Make Sqoop compatible with HBase 0.95+ (Jarek Jarcec Cecho via Venkat Ranganathan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a9089430 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a9089430 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a9089430 Branch: refs/heads/trunk Commit: a9089430e41abafdbd460009ad695f29613787fd Parents: e65d9c7 Author: Venkat Ranganathan <[email protected]> Authored: Tue Aug 13 10:51:00 2013 -0700 Committer: Venkat Ranganathan <[email protected]> Committed: Tue Aug 13 10:51:00 2013 -0700 ---------------------------------------------------------------------- build.xml | 47 +++++++++++++++++++++++++++++---- ivy.xml | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 118 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/a9089430/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 1a21288..39c6337 100644 --- a/build.xml +++ b/build.xml @@ -43,25 +43,57 @@ </else> </if> - <!-- Set variables per target hadoop version --> + <!-- + Set default HBase profile if not set + + HBase 0.95 has broken one big monolithic archive into smaller pieces, so + that the build dependencies are different between HBase 0.94 and 0.95. + In addition, HBase since 0.95 is providing artifacts compatible with Hadoop 2.0. + + For HBase 0.95 on Hadoop 2.0.0: + ant -Dhadoopversion=200 -Dhbaseprofile=95 + + For HBase 0.95 on Hadoop 1.0.0: + ant -Dhadoopversion=100 -Dhbaseprofile=95 + + For HBase 0.94 on Hadoop 1.0.0: + ant -Dhadoopversion=100 + + For HBase 0.94 on Hadoop 2.0.0: + - upstream HBase did not published Hadoop 2 compatible artifacts + --> + <if> + <isset property="hbaseprofile" /> + <then> + <echo message="Use HBase profile ${hbaseprofile}" /> + </then> + <else> + <echo message="Use HBase profile 0.94 by default" /> + <property name="hbaseprofile" value="94" /> + </else> + </if> + + <!-- Set dependency versions per target Hadoop version --> <if> <equals arg1="${hadoopversion}" arg2="20" /> <then> <property name="hadoop.version" value="0.20.2-cdh3u1" /> - <property name="hbase.version" value="0.90.3-cdh3u1" /> + <property name="hbase94.version" value="0.90.3-cdh3u1" /> <property name="zookeeper.version" value="3.3.3-cdh3u1" /> <property name="hadoop.version.full" value="0.20" /> <property name="hcatalog.version" value="0.11.0" /> + <property name="hbasecompatprofile" value="1" /> </then> <elseif> <equals arg1="${hadoopversion}" arg2="23" /> <then> <property name="hadoop.version" value="0.23.1" /> - <property name="hbase.version" value="0.92.0" /> + <property name="hbase94.version" value="0.92.0" /> <property name="zookeeper.version" value="3.4.2" /> <property name="hadoop.version.full" value="0.23" /> <property name="hcatalog.version" value="0.11.0" /> + <property name="hbasecompatprofile" value="2" /> </then> </elseif> @@ -69,10 +101,11 @@ <equals arg1="${hadoopversion}" arg2="100" /> <then> <property name="hadoop.version" value="1.0.0" /> - <property name="hbase.version" value="0.92.0" /> + <property name="hbase94.version" value="0.92.0" /> <property name="zookeeper.version" value="3.4.2" /> <property name="hadoop.version.full" value="1.0.0" /> <property name="hcatalog.version" value="0.11.0" /> + <property name="hbasecompatprofile" value="1" /> </then> </elseif> @@ -80,10 +113,11 @@ <equals arg1="${hadoopversion}" arg2="200" /> <then> <property name="hadoop.version" value="2.0.4-alpha" /> - <property name="hbase.version" value="0.94.2" /> + <property name="hbase94.version" value="0.94.2" /> <property name="zookeeper.version" value="3.4.2" /> <property name="hadoop.version.full" value="2.0.4-alpha" /> <property name="hcatalog.version" value="0.11.0" /> + <property name="hbasecompatprofile" value="2" /> </then> </elseif> @@ -92,6 +126,9 @@ </else> </if> + <!-- Set dependency versions that are working with all Hadoop versions--> + <property name="hbase95.version" value="0.95.2-hadoop${hbasecompatprofile}-SNAPSHOT" /> + <!-- Load system-wide and project-wide default properties set by the user, to avoid needing to override with -D. --> <property file="${user.home}/build.properties" /> http://git-wip-us.apache.org/repos/asf/sqoop/blob/a9089430/ivy.xml ---------------------------------------------------------------------- diff --git a/ivy.xml b/ivy.xml index 63fdc80..d7486e0 100644 --- a/ivy.xml +++ b/ivy.xml @@ -36,16 +36,19 @@ under the License. <conf name="common" visibility="private" extends="runtime" description="artifacts needed to compile/test the application"/> - <conf name="hbase" visibility="private" /> + <conf name="hbase94" visibility="private" /> + <conf name="hbase95" visibility="private" extends="hbasecompat${hbasecompatprofile}" /> + <conf name="hbasecompat1" visibility="private" /> + <conf name="hbasecompat2" visibility="private" /> <conf name="hcatalog" visibility="private" /> <conf name="hadoop23" visibility="private" - extends="common,runtime,hbase,hcatalog" /> + extends="common,runtime,hbase${hbaseprofile},hcatalog" /> <conf name="hadoop20" visibility="private" - extends="common,runtime,hbase,hcatalog" /> + extends="common,runtime,hbase${hbaseprofile},hcatalog" /> <conf name="hadoop100" visibility="private" - extends="common,runtime,hbase,hcatalog" /> + extends="common,runtime,hbase${hbaseprofile},hcatalog" /> <conf name="hadoop200" visibility="private" - extends="common,runtime,hbase,hcatalog" /> + extends="common,runtime,hbase${hbaseprofile},hcatalog" /> <conf name="test" visibility="private" extends="common,runtime"/> <conf name="hadoop23test" visibility="private" extends="test,hadoop23" /> @@ -164,8 +167,8 @@ under the License. <dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}" conf="releaseaudit->default"/> - <dependency org="org.apache.hbase" name="hbase" - rev="${hbase.version}" conf="hbase->default"> + <!-- HBase 0.94 --> + <dependency org="org.apache.hbase" name="hbase" rev="${hbase94.version}" conf="hbase94->default"> <artifact name="hbase" type="jar"/> <artifact name="hbase" type="test-jar" ext="jar" m:classifier="tests"/> <exclude org="com.sun.jersey" module="jersey-core"/> @@ -177,6 +180,72 @@ under the License. <exclude org="com.cloudera.cdh" module="zookeeper-ant" /> </dependency> + <!-- HBase 0.95 --> + <dependency org="org.apache.hbase" name="hbase-client" rev="${hbase95.version}" conf="hbase95->default"> + <artifact name="hbase-client" type="jar"/> + <artifact name="hbase-client" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-common" rev="${hbase95.version}" conf="hbase95->default"> + <artifact name="hbase-common" type="jar"/> + <artifact name="hbase-common" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-server" rev="${hbase95.version}" conf="hbase95->default"> + <artifact name="hbase-server" type="jar"/> + <artifact name="hbase-server" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-hadoop-compat" rev="${hbase95.version}" conf="hbase95->default"> + <artifact name="hbase-hadoop-compat" type="jar"/> + <artifact name="hbase-hadoop-compat" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-hadoop-compat" rev="${hbase95.version}" conf="hbase95->default"> + <artifact name="hbase-hadoop-compat" type="jar"/> + <artifact name="hbase-hadoop-compat" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-hadoop1-compat" rev="${hbase95.version}" conf="hbasecompat1->default"> + <artifact name="hbase-hadoop1-compat" type="jar"/> + <artifact name="hbase-hadoop1-compat" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + <dependency org="org.apache.hbase" name="hbase-hadoop2-compat" rev="${hbase95.version}" conf="hbasecompat2->default"> + <artifact name="hbase-hadoop2-compat" type="jar"/> + <artifact name="hbase-hadoop2-compat" type="test-jar" ext="jar" m:classifier="tests"/> + <exclude org="com.sun.jersey" module="jersey-core"/> + <exclude org="com.sun.jersey" module="jersey-json"/> + <exclude org="com.sun.jersey" module="jersey-server"/> + <exclude org="org.apache.thrift" module="thrift"/> + <exclude org="log4j" module="log4j"/> + </dependency> + + <dependency org="org.apache.hcatalog" name="hcatalog-core" rev="${hcatalog.version}" conf="hcatalog->default"> <artifact name="hcatalog-core" type="jar"/>
