Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Thu Apr 12 10:08:00 2012 @@ -52,52 +52,6 @@ sub new return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - $cmd =~ s/:OUTPATH:/$outfile/g; - $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; - $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g; - $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; - $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; - $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; - $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; - $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; - $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; -# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; -# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; -# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; - $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; - $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; - $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; - - if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; - } else { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; - } - - $cmd =~ s/:THRIFTSERVER:/$testCmd->{'thriftserver'}/g; - $cmd =~ s/:HADOOP_CLASSPATH:/$testCmd->{'hadoop_classpath'}/g; - $cmd =~ s/:HCAT_JAR:/$testCmd->{'hcatalog.jar'}/g; - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; @@ -127,10 +81,7 @@ sub globalSetup $ENV{'PATH'} = $globalHash->{'scriptPath'}; } - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); - - print $log "Going to run " . join(" ", @cmd) . "\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; + Util::runHadoopCmd($globalHash, $log, "fs -mkdir $globalHash->{'outpath'}"); IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or die "Cannot create localpath directory " . $globalHash->{'localpath'} . @@ -145,10 +96,7 @@ sub globalSetup die "Cannot create temporary directory " . $globalHash->{'tmpPath'} . " " . "$ERRNO\n"; - # Create the HDFS temporary directory - @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); - print $log "Going to run " . join(" ", @cmd) . "\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; + Util::runHadoopCmd($globalHash, $log, "fs -mkdir tmp/$globalHash->{'runid'}"); } sub globalCleanup @@ -199,8 +147,8 @@ sub runTest my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; # Copy result file out of hadoop - my @baseCmd = $self->getPigCmd($testCmd, $log); - my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); + my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $outputs[$i] = $testOut; $id++; } @@ -236,18 +184,16 @@ sub dumpPigTable # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; push(@cmd, $pigfile); # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; - IPC::Run::run(\@cmd, \undef, $log, $log) or - die "Failed running $pigfile\n"; + IPC::Run::run(\@cmd, \undef, $log, $log) or die "Failed running $pigfile\n"; $result{'rc'} = $? >> 8; @@ -255,23 +201,17 @@ sub dumpPigTable my $localoutfile; my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); return $outfile; } sub postProcessSingleOutputFile { - my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_; + my ($self, $outfile, $localdir, $testCmd, $log) = @_; my $subName = (caller(0))[3]; - my @baseCmd = @{$baseCmd}; - my @copyCmd = @baseCmd; - push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir)); - print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n"; + Util::runHadoopCmd($globalHash, $log, "fs -copyToLocal $outfile $localdir"); - IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n"; - - # Sort the result if necessary. Keep the original output in one large file. # Use system not IPC run so that the '*' gets interpolated by the shell. @@ -301,8 +241,6 @@ sub postProcessSingleOutputFile } sub runHadoop -# Being modified from runPig -# !!! Works, but need to add other arguments, like queue...??? { my ($self, $testCmd, $log) = @_; my $subName = (caller(0))[3]; @@ -313,7 +251,13 @@ sub runHadoop my $hadoopfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".hadoop"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $hadoopcmd = $self->replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log ); + # Get all of the additional jars we'll need. + my $additionalJars = Util::getHBaseLibs($testCmd, $log); #hbase before hive for precedence over bundled hbase + $additionalJars .= Util::getHiveLibs($testCmd, $log); + $additionalJars .= Util::getHCatLibs($testCmd, $log); + $testCmd->{'libjars'} = $additionalJars; + $testCmd->{'libjars'} =~ s/:/,/g; + my $hadoopcmd = Util::replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log ); # adjust for the leading and trailing new line often seen in the conf file's command directives $hadoopcmd =~ s/^\s*(.*?)\s*$/\1/s; @@ -338,16 +282,12 @@ sub runHadoop my $cp = $testCmd->{'hcatalog.jar'}; $cp =~ s/,/:/g; # Add in the hcat config file - $cp .= ":" . $testCmd->{'hive.conf.dir'}; + $cp .= ":" . $testCmd->{'hiveconf'}; + $cp .= ":" . $additionalJars; $ENV{'HADOOP_CLASSPATH'} = $cp; - if (defined($testCmd->{'hbaseconfigpath'})) { - $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}"; - } - - if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) { - $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; - $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'}; + if (defined($testCmd->{'hbaseconf'})) { + $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconf'}"; } # Add su user if provided @@ -373,10 +313,10 @@ sub runHadoop die "Failed running $script\n"; my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); if ($self->countStores($testCmd)==1) { @outputs = (); - $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $result{'outputs'} = \@outputs; } @@ -535,7 +475,7 @@ sub runPig my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; @@ -543,7 +483,8 @@ sub runPig # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + #my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; # Add option -l giving location for secondary logs @@ -580,7 +521,7 @@ sub runPig # single query if ($stores == 1) { if ($copyResults) { - $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); + $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log); $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile } else { $result{'output'} = "NO_COPY"; @@ -596,7 +537,7 @@ sub runPig # Copy result file out of hadoop my $testOut; if ($copyResults) { - $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log); + $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, $testCmd, $log); } else { $testOut = "NO_COPY"; } @@ -615,47 +556,6 @@ sub runPig return \%result; } -sub getPigCmd($$$) -{ - my ($self, $testCmd, $log) = @_; - - my @pigCmd; - - # set the PIG_CLASSPATH environment variable - my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'})); - $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'})); - $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'})); - # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined - $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'}); - $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne "")); - - # Set it in our current environment. It will get inherited by the IPC::Run - # command. - $ENV{'PIG_CLASSPATH'} = $pcp; - - @pigCmd = ("$testCmd->{'pigpath'}/bin/pig"); - - if (defined($testCmd->{'additionaljars'})) { - push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'}); - } - - if ($testCmd->{'exectype'} eq "local") { - push(@{$testCmd->{'java_params'}}, "-Xmx1024m"); - push(@pigCmd, ("-x", "local")); - } - - my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}"; - if (defined($testCmd->{'java_params'})) { - $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}}); - } - - $ENV{'PIG_OPTS'} = $opts; - - print $log "Returning Pig command " . join(" ", @pigCmd) . "\n"; - print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n"; - return @pigCmd; -} - sub compareSingleOutput { my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm Thu Apr 12 10:08:00 2012 @@ -53,35 +53,11 @@ sub new return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:HARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; my $subName = (caller(0))[3]; - # Set up values for the metastore - Util::setupHiveProperties($globalHash, $log); - # Setup the output path my $me = `whoami`; chomp $me; Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Thu Apr 12 10:08:00 2012 @@ -52,48 +52,6 @@ sub new return $self; } -sub replaceParameters -{ -##!!! Move this to Util.pm - - my ($self, $cmd, $outfile, $testCmd, $log) = @_; - - # $self - $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; - - # $outfile - $cmd =~ s/:OUTPATH:/$outfile/g; - - # $ENV - $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; - - # $testCmd - $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; - $cmd =~ s/:OUTPATH:/$outfile/g; - $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; - $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g; - $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; - $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; - $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; - $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; - $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; - $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; -# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; -# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; -# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; - $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; - $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; - $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; - - if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; - } else { - $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; - } - - return $cmd; -} - sub globalSetup { my ($self, $globalHash, $log) = @_; @@ -123,7 +81,7 @@ sub globalSetup $ENV{'PATH'} = $globalHash->{'scriptPath'}; } - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); + my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'}); print $log "Going to run " . join(" ", @cmd) . "\n"; @@ -143,7 +101,7 @@ sub globalSetup " " . "$ERRNO\n"; # Create the HDFS temporary directory - @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); + @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}"); print $log "Going to run " . join(" ", @cmd) . "\n"; IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; } @@ -157,7 +115,7 @@ sub globalCleanup " " . "$ERRNO\n"; # Cleanup the HDFS temporary directory - my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}"); + my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}"); print $log "Going to run " . join(" ", @cmd) . "\n"; IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n"; } @@ -221,7 +179,7 @@ sub runTest my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; # Copy result file out of hadoop - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); $outputs[$i] = $testOut; $id++; @@ -264,22 +222,16 @@ sub runPigCmdLine } # Write the pig script to a file. - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; close(FH); # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; - # Add option -l giving location for secondary logs - ##!!! Should that even be here? - my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log"; - push(@cmd, "-logfile"); - push(@cmd, $locallog); - # Add pig parameters if they're provided if (defined($testCmd->{'pig_params'})) { # Processing :PARAMPATH: in parameters @@ -333,7 +285,7 @@ sub runScript } # Write the script to a file - my $cmd = $self->replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log ); + my $cmd = Util::replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log ); open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n"; print FH $cmd . "\n"; @@ -363,88 +315,6 @@ sub runScript } -sub getPigCmd($$$) -{ - my ($self, $testCmd, $log) = @_; - - my @pigCmd; - - # set the PIG_CLASSPATH environment variable - my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'})); - $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'})); - $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'})); - # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined - $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'}); - $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne "")); - - # Set it in our current environment. It will get inherited by the IPC::Run - # command. - $ENV{'PIG_CLASSPATH'} = $pcp; - - @pigCmd = ("$testCmd->{'pigpath'}/bin/pig"); - - if (defined($testCmd->{'additionaljars'})) { - push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'}); - } - - if ($testCmd->{'exectype'} eq "local") { - push(@{$testCmd->{'java_params'}}, "-Xmx1024m"); - push(@pigCmd, ("-x", "local")); - } - - my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}"; - if (defined($testCmd->{'java_params'})) { - $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}}); - } - - $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts; - - print $log "Returning Pig command " . join(" ", @pigCmd) . "\n"; - print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n"; - return @pigCmd; -} - -sub dumpPigTable -{ - my ($self, $testCmd, $table, $log, $id) = @_; - my $subName = (caller(0))[3]; - - my %result; - - # Write the pig script to a file. - my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.pig"; - my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . "dump.out"; - - open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; - print FH "a = load '$table' using org.apache.hcatalog.pig.HCatLoader(); store a into '$outfile';\n"; - close(FH); - - - # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); - my @cmd = @baseCmd; - - push(@cmd, $pigfile); - - - # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; - print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; - - IPC::Run::run(\@cmd, \undef, $log, $log) or - die "Failed running $pigfile\n"; - $result{'rc'} = $? >> 8; - - - # Get results from the command locally - my $localoutfile; - my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out"; - my $stores = $self->countStores($testCmd); - - $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log); - return $outfile; -} - sub runPig { my ($self, $testCmd, $log, $copyResults, $noFailOnFail) = @_; @@ -456,7 +326,7 @@ sub runPig my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig"; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); + my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log ); open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n"; print FH $pigcmd . "\n"; @@ -464,7 +334,7 @@ sub runPig # Build the command - my @baseCmd = $self->getPigCmd($testCmd, $log); + my @baseCmd = Util::getPigCmd($testCmd, $log); my @cmd = @baseCmd; # Add option -l giving location for secondary logs @@ -485,8 +355,9 @@ sub runPig # Run the command - print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n"; print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n"; + print $log "With PIG_CLASSPATH set to $ENV{'PIG_CLASSPATH'}\n"; + print $log "and HADOOP_HOME set to $ENV{'HADOOP_HOME'}\n"; my $runrc = IPC::Run::run(\@cmd, \undef, $log, $log); Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Thu Apr 12 10:08:00 2012 @@ -27,12 +27,14 @@ package Util; use IPC::Run qw(run); +use strict; +use English; sub prepareHCat { my ($self, $testCmd, $log) = @_; my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out"; - my $hcatCmd = $self->replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log); + my $hcatCmd = replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log); my @hivefiles = (); my @outfiles = (); @@ -50,46 +52,6 @@ sub prepareHCat Util::runHCatCmdFromFile($testCmd, $log, $hivefiles[0]); } -############################################################################## -# Sub: setupHiveProperties -# -# Assure that necessary values are set in config in order to set Hive -# Java properties. -# -# Returns: -# Nothing -sub setupHiveProperties($$) -{ - my ($cfg, $log) = @_; - - # Set up values for the metastore - if (defined($cfg->{'metastore_thrift'}) && $cfg->{'metastore_thrift'} == 1) { - if (! defined $cfg->{'metastore_host'} || $cfg->{'metastore_host'} eq "") { - print $log "When using thrift, you must set the key " . - " 'metastore_host' to the machine your metastore is on\n"; - die "metastore_host is not set in existing.conf\n"; - } - - $cfg->{'metastore_connection'} = - "jdbc:$cfg->{'metastore_db'}://$cfg->{'metastore_host'}/hivemetastoredb?createDatabaseIfNotExist=true"; - - if (! defined $cfg->{'metastore_passwd'} || $cfg->{'metastore_passwd'} eq "") { - $cfg->{'metastore_passwd'} = 'hive'; - } - - if (! defined $cfg->{'metastore_port'} || $cfg->{'metastore_port'} eq "") { - $cfg->{'metastore_port'} = '9933'; - } - - $cfg->{'metastore_uri'} = - "thrift://$cfg->{'metastore_host'}:$cfg->{'metastore_port'}"; - } else { - $cfg->{'metastore_connection'} = - "jdbc:derby:;databaseName=metastore_db;create=true"; - $cfg->{'metastore_driver'} = "org.apache.derby.jdbc.EmbeddedDriver"; - } -} - sub getHadoopCmd { my ( $properties ) = @_; @@ -113,7 +75,8 @@ sub getHadoopCmd } push (@baseCmd, $cmd); - push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'}); + push (@baseCmd, '--config', $properties->{'hadoopconfdir'}) + if defined($properties->{'hadoopconfdir'}); return @baseCmd; } @@ -148,31 +111,11 @@ sub runHiveCmdFromFile($$;$$$$) $outfile = $log if (!defined($outfile)); $errfile = $log if (!defined($errfile)); - my @cmd = ("$cfg->{'hivehome'}/bin/hive"); - - # Add all of the modified properties we want to set -# push(@cmd, "--hiveconf", "hive.metastore.uris=$cfg->{'thriftserver'}"); -# push(@cmd, "--hiveconf", "hive.metastore.local=false"); - -# if( defined($cfg->{'metastore.principal'}) && ($cfg->{'metastore.principal'} =~ m/\S+/) -# && ($cfg->{'metastore.principal'} ne '${metastore.principal}')){ -# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=true", "--hiveconf", "hive.metastore.kerberos.principal=$cfg->{'metastore.principal'}"); -# } else { -# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false"); -# } - - $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}"; - - if (defined($cfg->{'hive.additionaljars'})) { - $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'}; - } - -# if (defined($cfg->{'hiveconf'})) { -# foreach my $hc (@{$cfg->{'hiveconf'}}) { -# push(@cmd, "--hiveconf", $hc); -# } -# } - + my @cmd = ($cfg->{'hivebin'}); + + $ENV{'HIVE_CONF_DIR'} = $cfg->{'hiveconf'}; + $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hcatshare'}; + if (defined($cfg->{'hivecmdargs'})) { push(@cmd, @{$cfg->{'hivecmdargs'}}); } @@ -230,26 +173,28 @@ sub runHCatCmdFromFile($$;$$$$) my ($cfg, $log, $sql, $outfile, $errfile, $noFailOnFail) = @_; if (!defined($ENV{'HADOOP_HOME'})) { - die "Cannot run hive when HADOOP_HOME environment variable is not set."; + die "Cannot run hcat when HADOOP_HOME environment variable is not set."; } $outfile = $log if (!defined($outfile)); $errfile = $log if (!defined($errfile)); # unset HADOOP_CLASSPATH - $ENV{'HADOOP_CLASSPATH'} = ""; - $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'}; +# $ENV{'HADOOP_CLASSPATH'} = ""; + $ENV{'HADOOP_CLASSPATH'} = $cfg->{'hbaseconf'}; + $ENV{'HCAT_CLASSPATH'} = Util::getHBaseLibs($cfg, $log); my @cmd; if (defined($sql)) { - @cmd = ("$cfg->{'hcathome'}/bin/hcat", "-f", $sql); + @cmd = ("$cfg->{'hcatbin'}", "-f", $sql); } else { - @cmd = ("$cfg->{'hcathome'}/bin/hcat"); + @cmd = ("$cfg->{'hcatbin'}"); } my $envStr; for my $k (keys(%ENV)) { - $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/); + $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/ || + $k =~ /HCAT/); } $envStr .= " "; print $log "Going to run hcat command [" . join(" ", @cmd) . @@ -327,198 +272,33 @@ sub show_call_stack { sub getPigCmd { - my $subName = (caller(0))[3]; - my $jarkey = shift; - my ( $properties ) = @_; - my $isPigSqlEnabled= 0; - my @baseCmd; - die "$0.$subName: null properties" if (! $properties ); - -show_call_stack(); - #UGLY HACK for pig sql support - if ( $jarkey =~ /testsql/ ) { - - $isPigSqlEnabled= 1; - $jarkey = "testjar"; - - } - - my $cmd; - if ( $properties->{'use-pig.pl'} ) { - # The directive gives that - # 1) the 'pig' command will be called, as opposed to java - # 2) the conf file has full control over what options and parameters are - # passed to pig. - # I.e. no parameters should be passed automatically by the script here. - # - # This allows for testing of the pig script as installed, and for testin of - # the pig script's options, including error testing. - -print 'use-pig.pl?????'; - - $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig"; - if ( ! -x "$cmd" ) { - print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n"; - $cmd = `which pig`; - chomp $cmd; - print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; - } - die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd ); - push (@baseCmd, $cmd ); - - if(defined($properties->{'additionaljars'})) { - push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); - } - $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'}; - - if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically - # !!! - return @baseCmd; - } - - } else { - $cmd="java"; + my ( $cfg, $log ) = @_; -print 'not use-pig.pl?????'; - # Set JAVA options + my @cmd = ("$cfg->{'pigbin'}"); - # User can provide only one of - # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>) - # "-c <cluster>" is allowed only in non local mode - if(defined($properties->{'cluster.name'})) { - # use provided cluster - @baseCmd = ($cmd, '-c', $properties->{'cluster.name'}); - } else { - die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) ); - die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e $properties->{$jarkey} ); - - # use user provided jar - my $classpath; - - if (defined $properties->{'jythonjar'}) { - $classpath = "$classpath:" . $properties->{'jythonjar'}; - } - if( $properties->{'exectype'} eq "local") { - # in local mode, we should not use - # any hadoop-site.xml - $classpath= "$classpath:" . $properties->{$jarkey}; - $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); - @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m'); - - } else { - - # non local mode, we also need to specify - # location of hadoop-site.xml - die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n" - if ( !defined( $properties->{'testconfigpath'} ) ); - die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n" - if (! -e $properties->{'testconfigpath'} ); - - $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'}; - $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); - $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) ); - @baseCmd = ($cmd, '-cp', $classpath ); - } - } - - # sets the queue, for exampel "grideng" - if(defined($properties->{'queue'})) { - push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'}); - } - - if(defined($properties->{'additionaljars'})) { - push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); - } - - if( ( $isPigSqlEnabled == 1 ) ){ - - if(defined($properties->{'metadata.uri'})) { - push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'}); - } - - if(defined($properties->{'metadata.impl'})) { - push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'}); - }else{ - push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables'); - } - } - - # Add howl support - if(defined($properties->{'howl.metastore.uri'})) { - push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'}); - } + # sets the queue, for exampel "grideng" + if(defined($cfg->{'queue'})) { + push( @cmd,'-Dmapred.job.queue.name='.$cfg->{'queue'}); + } - # Set local mode property - # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { - # Removed above 'if...' for Pig 8. - my $java=`which java`; - my $version=`file $java`; - if ( $version =~ '32-bit' ){ - push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32'); - } else { - push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64'); - } - # } - - - # Add user provided java options if they exist - if (defined($properties->{'java_params'})) { - push(@baseCmd, @{$properties->{'java_params'}}); - } + my $cp = Util::getHCatLibs($cfg, $log) . Util::getHiveLibsForPig($cfg, $log) . + Util::getHBaseLibs($cfg, $log); + push(@cmd, ('-Dpig.additional.jars='. $cp)); + $cp .= ':' . $cfg->{'hiveconf'}; + $cp .= ':' . $cfg->{'hbaseconf'}; + $ENV{'PIG_CLASSPATH'} = $cp; - if(defined($properties->{'hod'})) { - push( @baseCmd, '-Dhod.server='); - } - - # sets the permissions on the jobtracker for the logs - push( @baseCmd,'-Dmapreduce.job.acl-view-job=*'); + # sets the permissions on the jobtracker for the logs + push( @cmd,'-Dmapreduce.job.acl-view-job=*'); - # Add Main - push(@baseCmd, 'org.apache.pig.Main'); - # Set local mode PIG option - if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { - push(@baseCmd, '-x'); - push(@baseCmd, 'local'); - } - - # Set Pig SQL options - if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) { - - if ( defined($properties->{'testoutpath'}) ) { - push( @baseCmd, '-u' ); - push( @baseCmd, $properties->{'testoutpath'} ); - } - - push( @baseCmd, '-s' ); - push( @baseCmd, '-f' ); - } - - } # end else of if use-pig.pl - - - # Add -latest or -useversion - if ( $cmd =~ 'pig$' ) { - # Add -latest, or -useversion if 'current' is not target build - if ( defined($properties->{'pigTestBuildName'})) { - if ($properties->{'pigTestBuildName'} eq 'latest') { - push(@baseCmd, '-latest'); - } elsif ($properties->{'pigTestBuildName'} ne 'current') { - push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}"); - } - } - } elsif ( $cmd =~ 'java' ) { - - # is this ever used: ??? - # Add latest if it's there - if (defined($properties->{'latest'})) { - push(@baseCmd, '-latest'); - } + if ( defined($cfg->{'exectype'}) && $cfg->{'exectype'}=~ "local" ) { + push(@cmd, ('-x', 'local')); } - return @baseCmd; + return @cmd; } @@ -527,18 +307,18 @@ sub setLocale my $locale= shift; # $locale = "en_US.UTF-8" if ( !$locale ); $locale = "ja_JP.utf8" if ( !$locale ); - $ENV[LC_CTYPE]="$locale"; - $ENV[LC_NUMERIC]="$locale"; - $ENV[LC_TIME]="$locale"; - $ENV[LC_COLLATE]="$locale"; - $ENV[LC_MONETARY]="$locale"; - $ENV[LC_MESSAGES]="$locale"; - $ENV[LC_PAPER]="$locale"; - $ENV[LC_NAME]="$locale"; - $ENV[LC_ADDRESS]="$locale"; - $ENV[LC_TELEPHONE]="$locale"; - $ENV[LC_MEASUREMENT]="$locale"; - $ENV[LC_IDENTIFICATION]="$locale"; + $ENV['LC_CTYPE']="$locale"; + $ENV['LC_NUMERIC']="$locale"; + $ENV['LC_TIME']="$locale"; + $ENV['LC_COLLATE']="$locale"; + $ENV['LC_MONETARY']="$locale"; + $ENV['LC_MESSAGES']="$locale"; + $ENV['LC_PAPER']="$locale"; + $ENV['LC_NAME']="$locale"; + $ENV['LC_ADDRESS']="$locale"; + $ENV['LC_TELEPHONE']="$locale"; + $ENV['LC_MEASUREMENT']="$locale"; + $ENV['LC_IDENTIFICATION']="$locale"; } sub getLocaleCmd @@ -560,4 +340,154 @@ sub getLocaleCmd ."export LC_IDENTIFICATION=\"$locale\""; } +sub replaceParameters +{ + + my ($cmd, $outfile, $testCmd, $log) = @_; + + # $self +# $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g; + + # $outfile + $cmd =~ s/:OUTPATH:/$outfile/g; + + # $ENV + $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g; + + # $testCmd + $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g; + $cmd =~ s/:OUTPATH:/$outfile/g; + $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g; + $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g; + $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g; + $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g; + $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g; + $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g; + $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g; + $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g; +# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g; +# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g; +# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g; + $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g; + $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g; + $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g; + $cmd =~ s/:HCAT_JAR:/$testCmd->{'libjars'}/g; + + if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { + $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g; + } else { + $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g; + } + + return $cmd; +} + +sub getHiveLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + +# Pig needs a limited set of the Hive libs, since they include some of the same jars +# and we get version mismatches if it picks up all the libraries. +sub getHiveLibsForPig($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hive-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /libfb303.jar/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /libthrift.jar/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /datanucleus-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /jdo2-api-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /commons-dbcp-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; + /commons-pool-.*\.jar$/ && do { + $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; + }; +# /hbase-.*\.jar$/ && do { +# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; +# }; +# /zookeeper-.*\.jar$/ && do { +# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':'; +# }; + } + closedir(LIB); + return $cp; +} + +sub getHBaseLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hbaselibdir'}) or die "Cannot open $cfg->{'hbaselibdir'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hbase-.*\.jar$/ && do { + $cp .= $cfg->{'hbaselibdir'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + opendir(LIB, $cfg->{'zklibdir'}) or die "Cannot open $cfg->{'zklibdir'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /zookeeper.*\.jar$/ && do { + $cp .= $cfg->{'zklibdir'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + + +sub getHCatLibs($$) +{ + my ($cfg, $log) = @_; + + my $cp; + opendir(LIB, $cfg->{'hcatshare'}) or die "Cannot open $cfg->{'hcatshare'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hcatalog-[0-9].*\.jar$/ && do { + $cp .= $cfg->{'hcatshare'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + opendir(LIB, $cfg->{'hcatlib'}) or die "Cannot open $cfg->{'hcatlib'}, $!\n"; + my @jars = readdir(LIB); + foreach (@jars) { + /hbase-storage-handler.*\.jar$/ && do { + $cp .= $cfg->{'hcatlib'} . '/' . $_ . ':'; + }; + } + closedir(LIB); + return $cp; +} + + 1; Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl Thu Apr 12 10:08:00 2012 @@ -361,8 +361,8 @@ sub findHiveJars() die 'Please set $HIVE_HOME\n'; } - my $execJar = `ls ../../../../../hive/external/build/ql/hive-exec-*.jar`; - my $cliJar = `ls ../../../../../hive/external/build/cli/hive-cli-*.jar`; + my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`; + my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`; chomp $execJar; chomp $cliJar; return ($execJar, $cliJar); Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml Thu Apr 12 10:08:00 2012 @@ -22,8 +22,6 @@ <path id="udf-classpath"> <fileset file="../../../../../../build/hcatalog/*.jar" /> <fileset file="../../../../../../build/ivy/lib/hcatalog/*.jar" /> - <fileset file="../../../../../../hive/external/build/dist/lib/*.jar" /> - <fileset file="../../../../../../hive/external/build/hadoopcore/hadoop-0.20.1/*.jar" /> </path> <target name="init"> Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java (original) +++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Apr 12 10:08:00 2012 @@ -50,9 +50,9 @@ import org.junit.Test; public class TestHCatLoaderComplexSchema { - private static MiniCluster cluster = MiniCluster.buildCluster(); + //private static MiniCluster cluster = MiniCluster.buildCluster(); private static Driver driver; - private static Properties props; + //private static Properties props; private void dropTable(String tablename) throws IOException, CommandNeedRetryException{ driver.run("drop table "+tablename); @@ -87,8 +87,8 @@ public class TestHCatLoaderComplexSchema hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); driver = new Driver(hiveConf); SessionState.start(new CliSessionState(hiveConf)); - props = new Properties(); - props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name")); + //props = new Properties(); + //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name")); } @@ -180,7 +180,7 @@ public class TestHCatLoaderComplexSchema MockLoader.setData(tablename+"Input", data); try { createTable(tablename, tableSchema); - PigServer server = new PigServer(ExecType.LOCAL, props); + PigServer server = new PigServer(ExecType.LOCAL); server.setBatchOn(); server.registerQuery("A = load '"+tablename+"Input' using org.apache.hcatalog.pig.MockLoader() AS "+pigSchema+";"); Schema dumpedASchema = server.dumpSchema("A"); Modified: incubator/hcatalog/trunk/storage-handlers/hbase/build.xml URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/build.xml?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/storage-handlers/hbase/build.xml (original) +++ incubator/hcatalog/trunk/storage-handlers/hbase/build.xml Thu Apr 12 10:08:00 2012 @@ -75,7 +75,7 @@ <!-- ivy properteis set here --> <property name="ivy.repo.dir" value="${user.home}/ivyrepo" /> - <property name="ivy.dir" location="ivy" /> + <property name="ivy.dir" location="../../ivy" /> <loadproperties srcfile="${ivy.dir}/libraries.properties"/> <property name="asfrepo" value="https://repository.apache.org"/> <property name="asfsnapshotrepo" value="${asfrepo}/content/repositories/snapshots"/> @@ -103,29 +103,14 @@ <fileset dir="${ivy.lib.dir}" includes="*.jar"/> <fileset dir="${hcatalog.dir}/build/hcatalog" includes=" *.jar"/> <path refid="common.classpath"/> - <fileset dir="${hive.root}/build/hbase-handler" includes="*.jar"/> </path> <path id="test.classpath"> <pathelement location="${test.build.classes}" /> <pathelement location="${build.classes}" /> <pathelement location="conf"/> - <pathelement location="${hive.conf.dir}"/> - <!-- jars Hadoop depends on --> - <fileset dir="${hive.root}/build/hadoopcore/hadoop-${hadoop.version}/" > - <include name="**/lib/*.jar" /> - <include name="hadoop-test-*.jar" /> - </fileset> - <path refid="classpath"/> - <pathelement location="${handler.jar}"/> - <!-- jars Hive depends on --> - <fileset dir="${hive.root}/build/ivy/lib/default/"> - <include name="**/*.jar" /> - <exclude name="*hbase*.jar" /> - <exclude name="zookeeper*.jar" /> - <exclude name="guava*.jar" /> - <exclude name="slf4j*.jar" /> - </fileset> + <fileset dir="${ivy.lib.dir}" includes="*.jar"/> + <fileset dir="${hcatalog.dir}/build/hcatalog" includes=" *.jar"/> </path> <!-- @@ -184,6 +169,11 @@ <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings" pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" conf="common"/> <ivy:cachepath pathid="compile.classpath" conf="common"/> + <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="default"/> + <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings" + pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" + conf="default"/> + <ivy:cachepath pathid="compile.classpath" conf="default"/> </target> <target name="ivy-releaseaudit" depends="ivy-init" description="Resolve, Retrieve Ivy-managed artifacts for releaseaudit configuration"> Modified: incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml?rev=1325181&r1=1325180&r2=1325181&view=diff ============================================================================== --- incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml (original) +++ incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml Thu Apr 12 10:08:00 2012 @@ -47,10 +47,81 @@ <artifact name="zookeeper" type="test-jar" ext="jar" m:classifier="tests"/> </dependency> - <!-- hbase test dependencies --> - <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}" conf="common->master" /> - <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="common->master" /> - <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master" /> + <dependency org="org.apache.hadoop" name="hadoop-core" + rev="${hadoop-core.version}" conf="common->master" /> + <dependency org="org.apache.hadoop" name="hadoop-test" + rev="${hadoop-test.version}" conf="common->master" /> + <dependency org="org.apache.hive" name="hive-metastore" + rev="${hive.version}" conf="common->master"/> + <dependency org="org.apache.hive" name="hive-common" + rev="${hive.version}" conf="common->master"/> + <dependency org="org.apache.hive" name="hive-cli" + rev="${hive.version}" conf="common->master"/> + <dependency org="org.apache.hive" name="hive-exec" + rev="${hive.version}" conf="common->master"/> + <dependency org="junit" name="junit" rev="${junit.version}" + conf="common->master"/> + <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}" + conf="common->master"/> + <dependency org="commons-logging" name="commons-logging" + rev="${commons-logging.version}" conf="common->master"/> + <dependency org="commons-logging" name="commons-logging-api" + rev="${commons-logging.version}" conf="common->master"/> + <!-- hbase test dependencies --> + <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}" conf="common->master" /> + <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="common->master" /> + <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master" /> + + <!-- needed to run tests --> + <dependency org="commons-configuration" name="commons-configuration" + rev="${commons-configuration.version}" conf="default"/> + <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" + rev="${jackson.version}" conf="common->master"/> + <dependency org="org.codehaus.jackson" name="jackson-core-asl" + rev="${jackson.version}" conf="common->master"/> + <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j.version}" + conf="common->master"/> + <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j.version}" + conf="common->master"/> + <dependency org="log4j" name="log4j" rev="${log4j.version}" + conf="common->master"/> + <dependency org="javax.jdo" name="jdo2-api" rev="${jdo.version}" + conf="default"/> + <dependency org="org.datanucleus" name="datanucleus-core" + rev="${datanucleus-core.version}" conf="default"/> + <dependency org="org.datanucleus" name="datanucleus-connectionpool" + rev="${datanucleus-connectionpool.version}" conf="default"/> + <dependency org="org.datanucleus" name="datanucleus-enhancer" + rev="${datanucleus-enhancer.version}" conf="default"/> + <dependency org="org.datanucleus" name="datanucleus-rdbms" + rev="${datanucleus-rdbms.version}" conf="default"/> + <dependency org="commons-dbcp" name="commons-dbcp" rev="${commons-dbcp.version}" + conf="common->master"> + <exclude module="commons-pool" /> + <exclude org="org.apache.geronimo.specs" module="geronimo-jta_1.1_spec"/> + </dependency> + <dependency org="commons-pool" name="commons-pool" rev="${commons-pool.version}" + conf="default"/> + <dependency org="org.apache.derby" name="derby" rev="${derby.version}" + conf="default"/> + <dependency org="commons-configuration" name="commons-configuration" + rev="${commons-configuration.version}" conf="default"/> + <dependency org="commons-httpclient" name="commons-httpclient" + rev="${commons-httpclient.version}" conf="default"/> + <dependency org="org.apache.hive" name="hive-builtins" + rev="${hive.version}" conf="common->master"/> + <dependency org="org.mortbay.jetty" name="jetty" + rev="${jetty.version}" conf="default"/> + <dependency org="org.mortbay.jetty" name="jetty-util" + rev="${jetty.version}" conf="default"/> + <dependency org="org.apache.thrift" name="libfb303" rev="${fb303.version}" + conf="common->master"/> + <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" + conf="common->master" /> + <dependency org="org.apache.hive" name="hive-hbase-handler" + rev="${hive.version}" conf="common->master"/> + + </dependencies> </ivy-module>
