Author: hashutosh
Date: Fri Sep 9 20:39:36 2011
New Revision: 1167355
URL: http://svn.apache.org/viewvc?rev=1167355&view=rev
Log:
HCATALOG-97: Make test harness work with secure cluster
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Sep 9 20:39:36 2011
@@ -25,6 +25,8 @@ Trunk (unreleased changes)
NEW FEATURES
IMPROVEMENTS
+ HCAT-97. Make test harness work with secure cluster (hashutosh)
+
HCAT-94. Handle the job tracker tokens properly in har jobs (khorgath via
hashutosh)
HCAT-43. hcat.sh does not pick up hcatalog.jar correctly (macyang via
hashutosh)
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Fri Sep 9
20:39:36 2011
@@ -27,6 +27,7 @@
</fileset>
<fileset dir="${hive.dir}/build/dist/lib/">
<include name="*.jar" />
+ <exclude name="antlr*.jar" />
</fileset>
<pathelement location="${hcat.jar}" />
</path>
@@ -103,11 +104,7 @@
<fileset dir="${tool.src}/test"/>
<fileset dir="${tool.src}/generate"/>
</copy>
-<!--
- <copy todir="${tar.dir}/lib">
- <fileset dir="${lib.dir}"/>
- </copy>
--->
+
<copy todir="${tar.dir}/lib/java">
<fileset file="${udf.jar}"/>
</copy>
@@ -151,7 +148,7 @@
<target name="test" depends="property-check, udfs, tar, init-test">
<!-- If they have not specified tests to run then null it out -->
- <property name="tests.to.run" value=""/>
+ <property name="tests.to.run" value=""/>
<echo />
<exec executable="./test_harness.pl" dir="${test.location}">
<env key="HARNESS_ROOT" value="."/>
@@ -165,8 +162,8 @@
<env key="PH_OLDPIG" value="${harness.old.pig}"/>
<env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
<env key="HCAT_URL" value="${hcat.server.url}"/>
+ <env key="METASTORE_PRINCIPAL" value="${metastore.principal}"/>
<arg line="${tests.to.run}"/>
- <!-- <arg value="${test.location}/tests/bootstrap_hcat.conf"/> -->
<arg value="${test.location}/tests/hcat.conf"/>
</exec>
</target>
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Fri Sep 9
20:39:36 2011
@@ -63,7 +63,8 @@ $cfg = {
#HIVE
, 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin"
-
+
+ , 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}"
#HCATALOG
,'thriftserver' => "$ENV{HCAT_URL}"
,'hcatalog.jar' =>
"$ENV{HCAT_JAR},file://$ENV{HIVE_ROOT}/lib/thrift-fb303-0.5.0.jar,file://$ENV{HIVE_ROOT}/lib/thrift-0.5.0.jar,file://$ENV{HIVE_ROOT}/build/metastore/hive-metastore-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/common/hive-common-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/shims/hive-shims-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/serde/hive-serde-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/ql/hive-exec-0.8.0-SNAPSHOT.jar"
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
Fri Sep 9 20:39:36 2011
@@ -442,8 +442,14 @@ sub runHive
my @cmd = Util::getHiveCmd($testCmd);
#Add metastore info
- push(@cmd, " --hiveconf hive.metastore.local=false --hiveconf
hive.metastore.uris=thrift://".$testCmd->{'thriftserver'});
+ push(@cmd, "--hiveconf hive.metastore.local=false --hiveconf
hive.metastore.uris=thrift://".$testCmd->{'thriftserver'});
+
+ if(defined($testCmd->{'metastore.principal'})){
+ push(@cmd, "--hiveconf hive.metastore.sasl.enabled=true --hiveconf
hive.metastore.kerberos.principal=$testCmd->{'metastore.principal'}");
+
+ }
+
# Add hive command file
push(@cmd, '-f', $hivefile);
@@ -624,7 +630,11 @@ sub runHadoopCmdLine
my $hadoop_classpath = $self->replaceParameters(
$testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log );
$ENV{'HADOOP_CLASSPATH'} = $ENV{'HCAT_EXTRA_JARS'};
}
-
+ my $hadoop_opts =
"-Dhive.metastore.uris=thrift://".$testCmd->{'thriftserver'}."
-Dhcat.metastore.uri=thrift://".$testCmd->{'thriftserver'};
+ if (defined($testCmd->{'metastore.principal'})){
+ $hadoop_opts = join '',$hadoop_opts,"
-Dhive.metastore.sasl.enabled=true
-Dhcat.metastore.principal=",$testCmd->{'metastore.principal'},"
-Dhive.metastore.kerberos.principal=",$testCmd->{'metastore.principal'};
+ }
+ $ENV{'HADOOP_OPTS'} = $hadoop_opts;
# Run the command
print $log "$0:$subName Going to run command: $command\n";
print $log "$0:$subName STD OUT IS IN FILE: $stdoutfile\n";
@@ -840,7 +850,13 @@ sub runPig
my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" .
$testCmd->{'num'} . ".log";
push(@cmd, "-logfile");
push(@cmd, $locallog);
-
+
+ my $pig_opts =
"-Dhive.metastore.uris=thrift://".$testCmd->{'thriftserver'}."
-Dhcat.metastore.uri=thrift://".$testCmd->{'thriftserver'};
+ if (defined($testCmd->{'metastore.principal'})){
+ $pig_opts = join '',$pig_opts," -Dhive.metastore.sasl.enabled=true
-Dhcat.metastore.principal=",$testCmd->{'metastore.principal'},"
-Dhive.metastore.kerberos.principal=",$testCmd->{'metastore.principal'};
+ }
+ $ENV{'PIG_OPTS'} = $pig_opts;
+
# Add pig parameters if they're provided
if (defined($testCmd->{'pig_params'})) {
# Processing :PARAMPATH: in parameters
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Fri Sep 9
20:39:36 2011
@@ -231,7 +231,7 @@ sub getPigCmd
if(defined($properties->{'additionaljars'})) {
push(
@baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
}
-
+ $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'};
if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments
automatically
# !!!
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Fri Sep 9
20:39:36 2011
@@ -346,7 +346,7 @@ limit 5
{
'num' => 1,
'hive' => "show tables;",
- 'expected_out_regex' => "OK",
+ 'expected_err_regex' => "OK",
},
{
'num' => 5,
@@ -728,10 +728,10 @@ a = load 'default.boolean_table' using o
b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('boolean+int',
*);
store b into ':OUTPATH:';
",
- 'rc' => 0,
- 'expected_err_regex' => "Success",
+ 'rc' => 6,
+ 'expected_err_regex' => "ERROR",
# 'expected_out_regex' => "1",
- # 'not_expected_out_regex' => "[^1\\t]",
+ 'not_expected_out_regex' => "Success",
},
@@ -1427,7 +1427,7 @@ store A into 'default.tmp_pig2pig_stored
{
'num' => 3,
# complex stored by pig
- 'depends_on' => 'hcat_pig2pig_setup_tables_2', # not
really, but #4 does
+ # 'depends_on' => 'hcat_pig2pig_setup_tables_2', # not
really, but #4 does
'hive' => "
CREATE TABLE tmp_pig2pig_stored_complex_:RUNID: (
mymap map<string, string>,
@@ -2249,7 +2249,6 @@ store b into ':OUTPATH:';
'pig' => "
register :FUNCPATH:/testudf.jar;
a = load 'default.tmp_pig2pig_stored_complex_:RUNID:' using
org.apache.hcatalog.pig.HCatLoader();
--- b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('{mymap:
map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield:
map[])},rownum: int', *);
b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('mymap:
map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield:
map[])},rownum: int', *);
store b into ':OUTPATH:';
",
