Author: toffer
Date: Tue Jan 24 03:36:03 2012
New Revision: 1235122
URL: http://svn.apache.org/viewvc?rev=1235122&view=rev
Log:
HCAT-219. Adding HBase e2e tests (daijy via toffer)
Modified:
incubator/hcatalog/branches/branch-0.3/CHANGES.txt
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
Modified: incubator/hcatalog/branches/branch-0.3/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/CHANGES.txt?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.3/CHANGES.txt Tue Jan 24 03:36:03 2012
@@ -60,6 +60,8 @@ Release 0.3.0 (unreleased changes)
HCAT-63. RPM package integration with Hadoop (khorgath via hashutosh)
IMPROVEMENTS
+ HCAT-219. Adding HBase e2e tests (daijy via toffer)
+
HCAT-223. Include HWI (vikram.dixit via khorgath)
HCAT-179. Make HCatalog compile against Hadoop 0.23 (thw via khorgath)
Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml
(original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml Tue
Jan 24 03:36:03 2012
@@ -198,6 +198,7 @@
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PIG_HOME" value="${harness.pig.home}"/>
<env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without
antlr -->
+ <env key="HBASE_CONF_DIR" value="${hbase.conf.dir}"/>
<arg line="${tests.to.run}"/>
<arg value="${test.location}/tests/pig.conf"/>
<arg value="${test.location}/tests/hive.conf"/>
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
(original)
+++
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
Tue Jan 24 03:36:03 2012
@@ -85,6 +85,7 @@ $cfg = {
#HIVE
, 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin"
+ , 'hbaseconfigpath' => "$ENV{HBASE_CONF_DIR}"
, 'hivehome' => "$ENV{HIVE_HOME}"
};
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
(original)
+++
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
Tue Jan 24 03:36:03 2012
@@ -343,6 +343,10 @@ sub runHadoop
$ENV{'HADOOP_CLASSPATH'} = $cp;
}
+ if (defined($testCmd->{'hbaseconfigpath'})) {
+ $ENV{'HADOOP_CLASSPATH'} =
"$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
+ }
+
if (defined($testCmd->{'metastore.principal'}) &&
($testCmd->{'metastore.principal'} =~ m/\S+/)) {
$ENV{'HADOOP_OPTS'} = "-Dhcat.metastore.principal=" .
$testCmd->{'metastore.principal'};
$ENV{'HADOOP_CLIENT_OPTS'} = "-Dhcat.metastore.principal=" .
$testCmd->{'metastore.principal'};
@@ -624,6 +628,7 @@ sub getPigCmd($$$)
$pcp .= ":" . $testCmd->{'additionaljars'} if
(defined($testCmd->{'additionaljars'}));
# Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
$pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne
"local"); #&& (! defined $ENV{'HADOOP_HOME'});
+ $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne
"local" && defined($testCmd->{'hbaseconfigpath'} &&
$testCmd->{'hbaseconfigpath'} ne ""));
# Set it in our current environment. It will get inherited by the IPC::Run
# command.
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
(original)
+++
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
Tue Jan 24 03:36:03 2012
@@ -375,6 +375,7 @@ sub getPigCmd($$$)
$pcp .= ":" . $testCmd->{'additionaljars'} if
(defined($testCmd->{'additionaljars'}));
# Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
$pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne
"local"); #&& (! defined $ENV{'HADOOP_HOME'});
+ $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne
"local" && defined($testCmd->{'hbaseconfigpath'} &&
$testCmd->{'hbaseconfigpath'} ne ""));
# Set it in our current environment. It will get inherited by the IPC::Run
# command.
@@ -396,7 +397,7 @@ sub getPigCmd($$$)
$opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
}
- $ENV{'PIG_OPTS'} = $opts;
+ $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts;
print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and
PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
(original)
+++
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
Tue Jan 24 03:36:03 2012
@@ -562,7 +562,7 @@ sub findPigWithoutHadoopJar($$)
{
my ($cfg, $log) = @_;
- my $jar = `ls $cfg->{'pigpath'}/pig-*-withouthadoop.jar`;
+ my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
chomp $jar;
return $jar;
}
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
(original)
+++
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
Tue Jan 24 03:36:03 2012
@@ -158,5 +158,21 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
},
],
}, # end g
+ {
+ 'name' => 'Hadoop_HBase',
+ 'tests' => [
+ {
+ 'num' => 1
+ ,'hcat_prep'=>q\drop table if exists
hadoop_hbase_1;
+create table hadoop_hbase_1(key string, gpa string) STORED BY
'org.apache.hcatalog.hbase.HBaseHCatStorageHandler' TBLPROPERTIES
('hbase.columns.mapping'=':key,info:gpa');\
+ ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.HBaseReadWrite -libjars
:HCAT_JAR: :THRIFTSERVER: :INPATH:/studenttab10k hadoop_hbase_1 :OUTPATH:
+\,
+ ,'sql' => q\select name, sum(gpa) from
studenttab10k group by name;\
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ },
+ ],
+ }, # end g
]
}
Modified:
incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
(original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
Tue Jan 24 03:36:03 2012
@@ -174,6 +174,41 @@ store b into 'pig_write_3' using org.apa
}
],
}, # end g
+ {
+ 'name' => 'Pig_HBase',
+ 'tests' => [
+ {
+ 'num' => 1
+ ,'pig' => q\set hcat.hbase.output.bulkMode
'false'
+e = load 'pig_hbase_1' using org.apache.hcatalog.pig.HCatLoader();
+store e into ':OUTPATH:';\,
+ ,'result_table' => ['?']
+ ,'sql' => [ 'select name, avg(cast(age as
decimal(10,5))), avg(gpa) from studenttab10k group by name;']
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ },
+ {
+ # multiquery
+ 'num' => 2
+ ,'hcat_prep'=>q\drop table if exists
pig_hbase_2_1;
+create table pig_hbase_2_1(key string, age string, gpa string) STORED BY
'org.apache.hcatalog.hbase.HBaseHCatStorageHandler' TBLPROPERTIES
('hbase.columns.mapping'=':key,info:age,info:gpa');
+drop table if exists pig_hbase_2_2;
+create table pig_hbase_2_2(key string, age string, gpa string) STORED BY
'org.apache.hcatalog.hbase.HBaseHCatStorageHandler' TBLPROPERTIES
('hbase.columns.mapping'=':key,info:age,info:gpa');
+\
+ ,'pig' => q\set hcat.hbase.output.bulkMode
'false'
+a = load ':INPATH:/studenttab10k' as (name:chararray, age:int, gpa:float);
+b = group a by name;
+c = foreach b generate group as name, AVG(a.age) as age, AVG(a.gpa) as gpa;
+d = foreach c generate name as key, (chararray)age, (chararray)gpa as gpa;
+store d into 'pig_hbase_2_1' using org.apache.hcatalog.pig.HCatStorer();
+store d into 'pig_hbase_2_2' using org.apache.hcatalog.pig.HCatStorer();\,
+ ,'result_table' =>
['pig_hbase_2_1','pig_hbase_2_2']
+ ,'sql' => [ 'select name, avg(cast(age as
decimal(10,5))), avg(gpa) from studenttab10k group by name;', 'select name,
avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k group by name;']
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ }
+ ],
+ }, # end g
]
}
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
Tue Jan 24 03:36:03 2012
@@ -16,6 +16,6 @@
junit.version=3.8.1
ivy.version=2.2.0
rats-lib.version=0.5.1
-hbase.version=0.90.3
+hbase.version=0.90.5
zookeeper.version=3.4.0
thrift.version=0.7.0
