Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1163097&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (added) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Tue Aug 30 06:18:16 2011 @@ -0,0 +1,460 @@ +#!/usr/bin/env perl + +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +########################################################################### +# Class: Util +# +# A collection of helper subroutines. +# + + +package Util; + +############################################################################## +# Sub: localTime +# +# Returns: +# A string with the local time + +sub localTime() { + + my $retval = time(); + + my $local_time = gmtime( $retval); + + return $local_time; + +} + +############################################################################## +# Sub: formatedTime +# Returns the time with following format "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]" +# +# Returns: +# formated time + +sub formatedTime() { + + my @weekday = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"); + + my $retval = time(); + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time); + $year = $year + 1900; + $mon += 1; + return "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]\n"; + + +} + +############################################################################## +# Sub: execCmd +# Records the command in the log and prints it to stdout before executing. +# +# Paramaters: +# $log - The log object +# $subName - The name of the subroutine the message originated at +# $lineNo - The line number of the subroutine the message originated at +# $cmd - The command string to execute +# $msg - A string, the message to print +# $level - (optional)The logging level for the message: DEBUG, INFO, WARN, ERROR, FATAL +# defaults to DEBUG. +# +# Returns: +# An array containing the output from the executed command. +# + +sub execCmd() { + + my ( $log, $subName, $lineNo, $cmd, $level ) = @_ ; + + my $count = @_; + my $thisSubName = (caller(0))[3]; + + #Check for errors in arguments + if ( $count < 4 ){ + + if ( $log ) { + $log->msg( $level, $thisSubName, __LINE__ , "Invalid number of arguments, got $count=( @_ )" ); + + } else { + print "ERROR: $0 $thisSubName at ".__LINE__."Invalid number of arguments\n"; + + } + return 1; + } + + #Log command, execute commdand, return results + $level = "DEBUG" if ( !$level ); + $log->msg( $level, $subName, $lineNo , "$cmd"); + + my @result = `$cmd`; + + $log->msg( $level, $subName, $lineNo , "@cmd") if ( @cmd ); + + return @result; +} + +sub getHadoopCmd +{ + my ( $properties ) = @_; + + my $subName = (caller(0))[3]; + my @baseCmd; + + die "$0.$subName: null properties" if (! $properties ); + + my $cmd; + + $cmd = $properties->{'gridstack.root'} . "/hadoop/current/bin/hadoop"; + if ( ! -x "$cmd" ) { + print STDERR "\n$0::$subName WARNING: Can't find hadoop command: $cmd\n"; + $cmd = `which hadoop`; + chomp $cmd; + print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; + } + if ( ! -x "$cmd" ) { + die "\n$0::$subName FATAL: Hadoop command does not exist: $cmd\n"; + } + push (@baseCmd, $cmd); + + push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'}); + + return @baseCmd; +} + + +sub getHiveCmd +{ + my ( $properties ) = @_; + + my $subName = (caller(0))[3]; + my @baseCmd; + + die "$0.$subName: null properties" if (! $properties ); + + my $cmd; + + $cmd = $properties->{'hive_bin_location'} . "/hive"; + if ( ! -x "$cmd" ) { + die "\n$0::$subName FATAL: Hive command does not exist: $cmd\n"; + } + push (@baseCmd, $cmd); + +# push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'}); + + return @baseCmd; +} + +sub getHCatCmd +{ + my ( $properties ) = @_; + + my $subName = (caller(0))[3]; + my @baseCmd; + + die "$0.$subName: null properties" if (! $properties ); + + my $cmd; + + $cmd = $properties->{'hcat_bin_location'}; + if ( ! -x "$cmd" ) { + print STDERR "\n$0::$subName WARNING: Can't find hcat command: $cmd\n"; + $cmd = `which hcat.sh`; + chomp $cmd; + print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; + } + die "\n$0::$subName FATAL: hcat command does not exist: $cmd\n" if ( ! -x $cmd ); + $ENV{"hive.metastore.local"} = "false"; + $ENV{"hive.metastore.uris"} = "thrift://".$properties->{'thriftserver'}; + push (@baseCmd, $cmd); + return @baseCmd; +} + + + +sub getPigCmd +{ + my $subName = (caller(0))[3]; + my $jarkey = shift; + my ( $properties ) = @_; + my $isPigSqlEnabled= 0; + my @baseCmd; + die "$0.$subName: null properties" if (! $properties ); + + #UGLY HACK for pig sql support + if ( $jarkey =~ /testsql/ ) { + + $isPigSqlEnabled= 1; + $jarkey = "testjar"; + + } + + my $cmd; + if ( $properties->{'use-pig.pl'} ) { + # The directive gives that + # 1) the 'pig' command will be called, as opposed to java + # 2) the conf file has full control over what options and parameters are + # passed to pig. + # I.e. no parameters should be passed automatically by the script here. + # + # This allows for testing of the pig script as installed, and for testin of + # the pig script's options, including error testing. + + + $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig"; + if ( ! -x "$cmd" ) { + print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n"; + $cmd = `which pig`; + chomp $cmd; + print STDERR "$0::$subName WARNING: Instead using command: $cmd\n"; + } + die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd ); + push (@baseCmd, $cmd ); + + if(defined($properties->{'additionaljars'})) { + push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); + } + + + if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically + # !!! + return @baseCmd; + } + + } else { + $cmd="java"; + + # Set JAVA options + + # User can provide only one of + # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>) + # "-c <cluster>" is allowed only in non local mode + if(defined($properties->{'cluster.name'})) { + # use provided cluster + @baseCmd = ($cmd, '-c', $properties->{'cluster.name'}); + } else { + + die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) ); + die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e $properties->{$jarkey} ); + + # use user provided jar + my $classpath; + + if (defined $properties->{'jythonjar'}) { + $classpath = "$classpath:" . $properties->{'jythonjar'}; + } + if( $properties->{'exectype'} eq "local") { + # in local mode, we should not use + # any hadoop-site.xml + $classpath= "$classpath:" . $properties->{$jarkey}; + $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); + @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m'); + + } else { + + # non local mode, we also need to specify + # location of hadoop-site.xml + die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n" + if ( !defined( $properties->{'testconfigpath'} ) ); + die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n" + if (! -e $properties->{'testconfigpath'} ); + + $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'}; + $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) ); + $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) ); + @baseCmd = ($cmd, '-cp', $classpath ); + } + } + + # sets the queue, for exampel "grideng" + if(defined($properties->{'queue'})) { + push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'}); + } + + if(defined($properties->{'additionaljars'})) { + push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'}); + } + + if( ( $isPigSqlEnabled == 1 ) ){ + + if(defined($properties->{'metadata.uri'})) { + push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'}); + } + + if(defined($properties->{'metadata.impl'})) { + push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'}); + }else{ + push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables'); + } + } + + # Add howl support + if(defined($properties->{'howl.metastore.uri'})) { + push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'}); + } + + # Set local mode property + # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { + # Removed above 'if...' for Pig 8. + my $java=`which java`; + my $version=`file $java`; + if ( $version =~ '32-bit' ){ + push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32'); + } else { + push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64'); + } + # } + + + # Add user provided java options if they exist + if (defined($properties->{'java_params'})) { + push(@baseCmd, @{$properties->{'java_params'}}); + } + + if(defined($properties->{'hod'})) { + push( @baseCmd, '-Dhod.server='); + } + + # sets the permissions on the jobtracker for the logs + push( @baseCmd,'-Dmapreduce.job.acl-view-job=*'); + + + # Add Main + push(@baseCmd, 'org.apache.pig.Main'); + + # Set local mode PIG option + if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) { + push(@baseCmd, '-x'); + push(@baseCmd, 'local'); + } + + # Set Pig SQL options + if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) { + + if ( defined($properties->{'testoutpath'}) ) { + push( @baseCmd, '-u' ); + push( @baseCmd, $properties->{'testoutpath'} ); + } + + push( @baseCmd, '-s' ); + push( @baseCmd, '-f' ); + } + + } # end else of if use-pig.pl + + + # Add -latest or -useversion + if ( $cmd =~ 'pig$' ) { + # Add -latest, or -useversion if 'current' is not target build + if ( defined($properties->{'pigTestBuildName'})) { + if ($properties->{'pigTestBuildName'} eq 'latest') { + push(@baseCmd, '-latest'); + } elsif ($properties->{'pigTestBuildName'} ne 'current') { + push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}"); + } + } + } elsif ( $cmd =~ 'java' ) { + + # is this ever used: ??? + # Add latest if it's there + if (defined($properties->{'latest'})) { + push(@baseCmd, '-latest'); + } + } + + return @baseCmd; +} + + +sub getBasePigSqlCmd +{ + + my $subName = (caller(0))[3]; + + Util::getPigCmd( 'testsql', @_ ); + +} + +sub getBasePigCmd +{ + + my $subName = (caller(0))[3]; + + Util::getPigCmd( 'testjar', @_ ); + +} + +sub getLatestBasePigCmd +{ + + my $subName = (caller(0))[3]; + + Util::getPigCmd( 'latesttestjar', @_ ); + +} + + +sub getBenchmarkBasePigCmd +{ + my $subName = (caller(0))[3]; + my ( $properties ) = @_; + + Util::getPigCmd( 'benchmarkjar', @_ ); + +} + +sub setLocale +{ + my $locale= shift; +# $locale = "en_US.UTF-8" if ( !$locale ); +$locale = "ja_JP.utf8" if ( !$locale ); + $ENV[LC_CTYPE]="$locale"; + $ENV[LC_NUMERIC]="$locale"; + $ENV[LC_TIME]="$locale"; + $ENV[LC_COLLATE]="$locale"; + $ENV[LC_MONETARY]="$locale"; + $ENV[LC_MESSAGES]="$locale"; + $ENV[LC_PAPER]="$locale"; + $ENV[LC_NAME]="$locale"; + $ENV[LC_ADDRESS]="$locale"; + $ENV[LC_TELEPHONE]="$locale"; + $ENV[LC_MEASUREMENT]="$locale"; + $ENV[LC_IDENTIFICATION]="$locale"; +} + +sub getLocaleCmd +{ + my $locale= shift; + $locale = "en_US.UTF-8" if ( !$locale ); + + return "export LC_CTYPE=\"$locale\";" + ."export LC_NUMERIC=\"$locale\";" + ."export LC_TIME=\"$locale\";" + ."export LC_COLLATE=\"$locale\";" + ."export LC_MONETARY=\"$locale\";" + ."export LC_MESSAGES=\"$locale\";" + ."export LC_PAPER=\"$locale\";" + ."export LC_NAME=\"$locale\";" + ."export LC_ADDRESS=\"$locale\";" + ."export LC_TELEPHONE=\"$locale\";" + ."export LC_MEASUREMENT=\"$locale\";" + ."export LC_IDENTIFICATION=\"$locale\""; +} +1;
Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3 URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3?rev=1163097&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3 (added) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3 Tue Aug 30 06:18:16 2011 @@ -0,0 +1,3 @@ +# this is a comment +fname = studenttab10k +unusedkey = unusedvalueihope
