Author: gates
Date: Tue May 29 23:51:41 2012
New Revision: 1344053
URL: http://svn.apache.org/viewvc?rev=1344053&view=rev
Log:
HCATALOG-414 More HCat e2e tests
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1344053&r1=1344052&r2=1344053&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue May 29 23:51:41 2012
@@ -26,6 +26,8 @@ Trunk (unreleased changes)
HCAT-328 HCatLoader should report its input size so pig can estimate the
number of reducers (traviscrawford via gates)
IMPROVEMENTS
+ HCAT-414 More HCat e2e tests (khorgath via gates)
+
HCAT-374 Add HCatBaseTest and one example usage (traviscrawford via gates)
HCAT-390 Add e2e test for Hive with different file formats in different
partitions (daijy via gates)
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm?rev=1344053&r1=1344052&r2=1344053&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
Tue May 29 23:51:41 2012
@@ -135,7 +135,7 @@ sub runHCatCmdLine
open(FH, "> $hcatfiles[0]") or
die "Unable to open file $hcatfiles[0] to write SQL script, $ERRNO\n";
- print FH $testCmd->{'hcat'} . "\n";
+ print FH $hcatCmd . "\n";
close(FH);
Util::runHCatCmdFromFile($testCmd, $log, $hcatfiles[0], $stdoutfile,
$stderrfile, 1);
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1344053&r1=1344052&r2=1344053&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Tue May 29
23:51:41 2012
@@ -64,7 +64,7 @@ describe extended hcat_createtable_2;
{
'num' => 3
,'hcat' => q\
-drop table if exists hcat_create_table_3;
+drop table if exists hcat_createtable_3;
create table if not exists hcat_createtable_3(name string, age int, gpa
double) stored as textfile;
create table if not exists hcat_createtable_3(name string, age int, gpa
double) stored as textfile;
describe hcat_createtable_3;
@@ -72,8 +72,54 @@ describe hcat_createtable_3;
,'rc' => 0
,'expected_out_regex' => 'name string(\s)*age
int(\s)*gpa double'
,'expected_err_regex' => 'OK(.*)OK(.*)OK(.*)OK'
+ },
+
+ {
+ 'num' => 4
+ ,'hcat' => q\
+drop table if exists hcat_createtable_4;
+create table hcat_createtable_4(name string, age int, gpa double) partitioned
by (grade string, section string) stored as textfile;
+alter table hcat_createtable_4 add partition (grade='7',section='a');
+alter table hcat_createtable_4 add partition (grade='8',section='b');
+show partitions hcat_createtable_4;
+\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'grade=7/section=a(\s)*grade=8/section=b'
+ ,'expected_err_regex' => 'OK(.*)OK(.*)OK'
+ },
+ {
+ 'num' => 5
+ ,'hcat' => q\
+drop table if exists hcat_createtable_5;
+create table hcat_createtable_5(name string, age int, gpa double) stored as
+inputformat 'org.apache.hadoop.hive.ql.io.RCFileInputFormat'
+outputformat 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat';
+describe extended hcat_createtable_5;
+\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'name string(\s)*age
int(\s)*gpa
double(.*(\s))*inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat,
outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
+ ,'expected_err_regex' => 'OK(.*)OK(.*)OK'
+ },
+
+ {
+ 'num' => 6
+ ,'hcat' => q#
+drop table if exists hcat_createtable_6;
+create table hcat_createtable_6(a int, b array<int>, c map<string, string>)
+row format delimited
+ fields terminated by '^'
+ collection items terminated by ','
+ map keys terminated by ':'
+ lines terminated by '\\n'
+stored as textfile;
+describe extended hcat_createtable_6;
+#,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'a int(\s)*b
array<int>(\s)*c
map<string,string>(.*(\s))*.*parameters:.*colelction.delim=,, mapkey.delim=:,
serialization.format=|, line.delim=\n, field.delim=^'
+ ,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
+
],
}, # end g
{
@@ -104,6 +150,41 @@ describe hcat_droptable_2;\,
,'expected_out_regex' => 'does not exist'
},
+ {
+ 'num' => 3
+ ,'hcat' => q\
+drop table if exists hcat_drop_table_4;
+dfs -rmr :OUTPATH:/../drop_table_ext;
+dfs -cp :INPATH:/studentnull10k/ :OUTPATH:/../drop_table_ext;
+\,
+ },
+ {
+ 'num' => 4
+ ,'depends_on' => 'HCat_DropTable_3'
+ ,'hcat' => q\
+create external table hcat_drop_table_4(name string, age int, gpa double)
stored as textfile location 'hdfs://:OUTPATH:/../drop_table_ext';
+describe extended hcat_drop_table_4;
+\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'(.*)location:hdfs://(.*)drop_table_ext(.*)tableType:EXTERNAL_TABLE'
+ },
+ {
+ 'num' => 5
+ ,'depends_on' => 'HCat_DropTable_4'
+ ,'hcat' => q\
+drop table hcat_drop_table_4;
+\,
+ ,'rc' => 0
+ },
+ {
+ 'num' => 6
+ ,'depends_on' => 'HCat_DropTable_5'
+ ,'hcat' => q\
+dfs -ls :OUTPATH:/../drop_table_ext
+\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'(.*(\s))*.*drop_table_ext/studentnull10k'
+ },
],
}, # end g
{
@@ -164,6 +245,121 @@ alter table hcat_altertable_6 set serde
\
,'rc' => 0
},
+ {
+ 'num' => 7
+ ,'hcat' => q\
+drop table if exists hcat_altertable_7;
+create table hcat_altertable_7(name string, age int, gpa double) partitioned
by (grade string, section string) stored as textfile;
+alter table hcat_altertable_7 add partition (grade='7',section='a');
+show partitions hcat_altertable_7;\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'grade=7/section=a'
+ },
+ {
+ 'num' => 8
+ ,'depends_on' => 'HCat_AlterTable_7'
+ ,'hcat' => q\
+alter table hcat_altertable_7 add partition (grade='7',section='b');
+alter table hcat_altertable_7 add partition (grade='7',section='c');
+show partitions hcat_altertable_7;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c'
+ },
+ {
+ 'num' => 9
+ ,'depends_on' => 'HCat_AlterTable_8'
+ ,'hcat' => q\
+alter table hcat_altertable_7 add if not exists partition
(grade='7',section='a');\
+ ,'rc' => 0
+ },
+
+ {
+ 'num' => 10
+ ,'depends_on' => 'HCat_AlterTable_9'
+ ,'hcat' => q\
+alter table hcat_altertable_7 add partition (grade='8',section='a') partition
(grade='8',section='b');
+show partitions hcat_altertable_7;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c(\s)*grade=8/section=a(\s)*grade=8/section=b'
+ },
+ {
+ 'num' => 11
+ ,'depends_on' => 'HCat_AlterTable_10'
+ ,'hcat' => q\
+alter table hcat_altertable_7 drop partition (grade='8',section='a'),
partition (grade='8',section='b');
+show partitions hcat_altertable_7;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'grade=7/section=a(\s)*grade=7/section=b(\s)*grade=7/section=c'
+ ,'not_expected_out_regex' =>
'grade=8/section=a(\s)*grade=8/section=b'
+ },
+ {
+ 'num' => 12
+ ,'depends_on' => 'HCat_AlterTable_11'
+ ,'hcat' => q\
+set hive.exec.drop.ignorenonexistent = false;
+alter table hcat_altertable_7 drop if exists partition
(grade='8',section='a'), partition(grade='7',section='d'),
partition(grade='7',section='c');
+set hive.exec.drop.ignorenonexistent = true;
+show partitions hcat_altertable_7;\,
+ ,'expected_out_regex' =>
'grade=7/section=a(\s)*grade=7/section=b'
+ ,'rc' => 0
+ },
+ {
+ 'num' => 13
+ ,'hcat' => q\
+drop table if exists hcat_altertable_13;
+create table hcat_altertable_13(a int, b string) partitioned by (c string, d
string) stored as textfile;
+alter table hcat_altertable_13 add partition (c='1',d='1');
+alter table hcat_altertable_13 add columns (x string comment 'extra');
+describe hcat_altertable_13;\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'a int(\s)*b
string(\s)*x string extra(\s)*c string(\s)*d string'
+ },
+ {
+ 'num' => 14
+ ,'depends_on' => 'HCat_AlterTable_13'
+ ,'hcat' => q\
+alter table hcat_altertable_13 replace columns (k int, l string, m string);
+describe hcat_altertable_13;\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'k int(\s)*l
string(\s)*m string(\s)*c string(\s)*d string'
+ },
+ {
+ 'num' => 15
+ ,'hcat' => q\
+drop table if exists hcat_altertable_15b;
+drop table if exists hcat_altertable_15a;
+create table hcat_altertable_15a(a int, b string) stored as textfile;
+alter table hcat_altertable_15a rename to hcat_altertable_15b;
+show tables like 'hcat_altertable_15.*';\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'hcat_altertable_15b'
+ ,'not_expected_out_regex' =>
'hcat_altertable_15a'
+ },
+
+ {
+ 'num' => 16
+ ,'hcat' => q\
+drop table if exists hcat_altertable_16;
+create table hcat_altertable_16(a int, b string) stored as textfile;
+show table extended like hcat_altertable_16;
+;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'((.*)\s)*location(.*)hcat_altertable_16'
+ },
+
+ {
+ 'num' => 17
+ ,'depends_on' => 'HCat_AlterTable_16'
+ ,'hcat' => q\
+alter table hcat_altertable_16 set location 'hdfs:///tmp/table_has_moved/';
+show table extended like hcat_altertable_16;
+;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'location:hdfs:///tmp/table_has_moved'
+ ,'not_expected_out_regex' =>
'location(.*)hcat_altertable_16'
+ },
+
+
],
}, # end g
{
@@ -193,7 +389,7 @@ drop database hcat_database_1;\
drop table if exists hcat_view_1;
drop view if exists hcat_view_1_1;
drop view if exists hcat_view_1_2;
-create external table hcat_view_1 (name string, age int, gpa double) row
format delimited fields terminated by '\t' stored as TEXTFILE location
'/user/hcat/tests/data/studenttab10k';
+create external table hcat_view_1 (name string, age int, gpa double) row
format delimited fields terminated by '\t' stored as TEXTFILE location
':INPATH:/studenttab10k';
create view hcat_view_1_1 as select name, gpa, age from studenttab10k;
create view hcat_view_1_2 partitioned on (age) as select name, gpa, age from
studenttab10k;
alter view hcat_view_1_1 set tblproperties('key'='value');
@@ -204,7 +400,17 @@ drop view hcat_view_1_1;
drop view hcat_view_1_3;"
,'rc' => 0
},
- ],
+ {
+ 'num' => 2
+ ,'hcat' => q"
+drop view if exists hcat_view_2_1;
+create view hcat_view_2_1(name,age) as select name, age from studenttab10k;
+describe extended hcat_view_2_1;
+"
+ ,'expected_out_regex' => 'name
string.*(\s)age int(.*(\s))*viewOriginalText:select name, age from
studenttab10k.*tableType:VIRTUAL_VIEW'
+ ,'rc' => 0
+ },
+ ],
}, # end g
{
'name' => 'HCat_Authorize',
@@ -240,10 +446,43 @@ create table hcat_index_1 (a string) par
create index hcat_index_1_1 on table hcat_index_1(a) as 'compact' with
deferred rebuild comment 'hcat test';
alter index hcat_index_1_1 on hcat_index_1 set idxproperties ('prop1'='val1');
show indexes on hcat_index_1;
+create index hcat_index_1_2 on table hcat_index_1(a) as 'compact' with
deferred rebuild comment 'hcat test second';
+alter index hcat_index_1_2 on hcat_index_1 set idxproperties ('prop1'='val2');
+show indexes on hcat_index_1;
drop index hcat_index_1_1 on hcat_index_1;
+drop index hcat_index_1_2 on hcat_index_1;
;\
,'rc' => 0
},
+
+ {
+ 'num' => 2
+ ,'hcat' => q\
+drop index if exists hcat_index_2_1 on hcat_index_2;
+drop table if exists hcat_index_2;
+create table hcat_index_2 (a string) partitioned by (b string) stored as
TEXTFILE;
+create index hcat_index_2_1 on table hcat_index_2(a) as 'compact' with
deferred rebuild comment 'hcat compact idx';
+alter index hcat_index_2_1 on hcat_index_2 set idxproperties ('prop1'='val1');
+show indexes on hcat_index_2;
+;\
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'hcat_index_2_1(\s)*hcat_index_2(\s)*a(\s)*default__hcat_index_2_hcat_index_2_1__(\s)*compact(\s)*hcat
compact idx'
+ },
+
+ {
+ 'num' => 3
+ ,'hcat' => q\
+drop index if exists hcat_index_3_1 on hcat_index_3;
+drop table if exists hcat_index_3;
+create table hcat_index_3 (a string) partitioned by (b string) stored as
TEXTFILE;
+create index hcat_index_3_1 on table hcat_index_3(a) as 'bitmap' with deferred
rebuild comment 'hcat bitmap idx';
+alter index hcat_index_3_1 on hcat_index_3 set idxproperties ('prop1'='val1');
+show indexes on hcat_index_3;
+;\
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'hcat_index_3_1(\s)*hcat_index_3(\s)*a(\s)*default__hcat_index_3_hcat_index_3_1__(\s)*bitmap(\s)*hcat
bitmap idx'
+ },
+
],
}, # end g
{
@@ -264,6 +503,145 @@ unlock table studenttab10k;
],
}, # end g
{
+ 'name' => 'HCat_ShowDes',
+ 'tests' => [
+ {
+ 'num' => 1
+ ,'hcat' => q\
+drop table if exists hcat_showdes.hcat_showdestable_1;
+drop database if exists hcat_showdes;
+drop database if exists t_showdes_abc;
+drop table if exists hcat_showdestable_2;
+create database hcat_showdes;
+create database t_showdes_abc;
+create table hcat_showdes.hcat_showdestable_1 (a int, b string);
+create table hcat_showdestable_2 (a int comment 'first', b struct< foo : int,
bar : struct< bar1:int, bar2:int > > comment 'second') partitioned by (c
string);
+alter table hcat_showdestable_2 add partition (c = '41');
+alter table hcat_showdestable_2 add partition (c = '42');
+;\
+ ,'rc' => 0
+ },
+ {
+ 'num' => 2
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show databases;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'hcat_showdes'
+ },
+ {
+ 'num' => 3
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show databases like '.*abc';\
+ ,'rc' => 0
+ ,'expected_out_regex' => 't_showdes_abc'
+ ,'not_expected_out_regex' => 'hcat_showdes'
+ },
+ {
+ 'num' => 4
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show tables;\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'hcat_showdestable_2'
+ },
+
+ {
+ 'num' => 5
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show tables in hcat_showdes;\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'hcat_showdestable_1'
+ ,'not_expected_out_regex' =>
'hcat_createtable_1'
+ },
+ {
+ 'num' => 6
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show tables in hcat_showdes like '.*table_1';\,
+ ,'rc' => 0
+ ,'expected_out_regex' => 'hcat_showdestable_1'
+ },
+ {
+ 'num' => 7
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show table extended like hcat_showdestable_2;\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)'
+ },
+ {
+ 'num' => 8
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show table extended like hcat_showdestable_2 partition(c='42');\,
+ ,'rc' => 0
+ ,'expected_out_regex' =>
'tableName(.*(\s))*owner(.*(\s))*location(.*(\s))*columns(.*(\s))*totalNumberFiles(.*(\s))*totalFileSize(.*(\s))*maxFileSize(.*(\s))*minFileSize(.*(\s))*lastAccessTime(.*(\s))*lastUpdateTime(.*)'
+ },
+ {
+ 'num' => 9
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+describe extended hcat_showdestable_2 partition(c='42');\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'a int
first(\s)*b struct<foo:int,bar:struct<bar1:int,bar2:int>> second(\s)*c
string(.*(\s))*Detailed Partition Information(\s)*Partition(.*)values:\[42\]'
+ },
+ {
+ 'num' => 10
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+describe extended hcat_showdestable_2.b;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'foo int from
deserializer(\s)*bar struct<bar1:int,bar2:int> from deserializer'
+ },
+ {
+ 'num' => 11
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+describe extended hcat_showdestable_2.b.foo;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'foo int from
deserializer'
+ ,'not_expected_out_regex' => 'bar'
+ },
+ {
+ 'num' => 12
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+describe extended hcat_showdestable_2.b.bar;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'bar1 int from
deserializer(\s)*bar2 int from deserializer'
+ },
+ {
+ 'num' => 13
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+describe extended hcat_showdestable_2.b.bar.bar1;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'bar1 int from
deserializer'
+ ,'not_expected_out_regex' => 'bar2'
+ },
+ {
+ 'num' => 14
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show partitions hcat_showdestable_2;\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'c=41(\s)*c=42'
+ },
+ {
+ 'num' => 15
+ ,'depends_on' => 'HCat_ShowDes_1'
+ ,'hcat' => q\
+show partitions hcat_showdestable_2 partition(c='41');\
+ ,'rc' => 0
+ ,'expected_out_regex' => 'c=41'
+ ,'not_expected_out_regex' => 'c=42'
+ },
+ ],
+ }, # end g
+ {
'name' => 'HCat_Misc',
'tests' => [
{
