Author: gates
Date: Sat Feb 25 01:58:06 2012
New Revision: 1293514
URL: http://svn.apache.org/viewvc?rev=1293514&view=rev
Log:
HCATALOG-257 e2e harness not working properly after file location change
Added:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/bin/hcat-config.sh
incubator/hcatalog/trunk/build.xml
incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sat Feb 25 01:58:06 2012
@@ -50,6 +50,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-257 e2e harness not working properly after file location change (gates)
+
HCAT-270 HCatRecord SerDe does not handle List<Map> or List<List> or
List<lazy primitive> correctly (khorgath via gates)
HCAT-248 Set execute.setugi conf variable to true by default (hashutosh via
gates)
Modified: incubator/hcatalog/trunk/bin/hcat-config.sh
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/bin/hcat-config.sh?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/bin/hcat-config.sh (original)
+++ incubator/hcatalog/trunk/bin/hcat-config.sh Sat Feb 25 01:58:06 2012
@@ -48,8 +48,8 @@ then
fi
# Allow alternate conf dir location.
-if [ -e "${HCAT_PREFIX}/conf/hcat-env.sh" ]; then
- DEFAULT_CONF_DIR=${HCAT_PREFIX}/"conf"
+if [ -e "${HCAT_PREFIX}/etc/hcatalog/hcat-env.sh" ]; then
+ DEFAULT_CONF_DIR=${HCAT_PREFIX}/"etc/hcatalog"
else
DEFAULT_CONF_DIR="/etc/hcatalog"
fi
Modified: incubator/hcatalog/trunk/build.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Sat Feb 25 01:58:06 2012
@@ -71,6 +71,7 @@
<property name="test.output" value="no"/>
<property name="hive.conf.dir" value="${hive.root}/conf"/>
<property name="test.warehouse.dir" value="/tmp/hcat_junit_warehouse"/>
+ <property name="test.excludes" value="${test.src.dir}/e2e/**"/>
<!-- e2e test properties -->
<property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
@@ -299,7 +300,7 @@
-->
<!-- Build HCatalog test files -->
<target name="compile-test" depends="jar">
- <javac encoding="${build.encoding}" srcdir="${test.src.dir}"
excludes="${excludes}"
+ <javac encoding="${build.encoding}" srcdir="${test.src.dir}"
excludes="${test.excludes}"
includes="**/*.java" destdir="${test.build.classes}"
debug="${javac.debug}"
optimize="${javac.optimize}" target="${javac.version}"
source="${javac.version}" deprecation="${javac.deprecation}"
@@ -666,6 +667,7 @@
<exclude name="ivy/*.jar"/>
<include name="lib/**"/>
<include name="license/**"/>
+ <include name="shims/**"/>
<include name="src/**"/>
<include name="storage-drivers/**"/>
<include name="*.txt"/>
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Sat Feb 25
01:58:06 2012
@@ -19,7 +19,7 @@
<property name="hcat.dir" value="${basedir}/../../../../"/>
<property name="hive.dir" value="${basedir}/../../../../hive/external/"/>
- <property name="hcat.jar"
value="${hcat.dir}/build/hcatalog/hcatalog-0.3.0-dev.jar"/>
+ <property name="hcat.jar"
value="${hcat.dir}/build/hcatalog/hcatalog-${hcatalog.version}.jar"/>
<path id="hcat.deps">
<fileset dir="${hive.dir}/lib/">
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Sat Feb 25
01:58:06 2012
@@ -62,7 +62,7 @@ $cfg = {
, 'hcat_data_dir' => '/user/hcat/tests/data'
, 'hivehome' => $ENV{'PH_HIVE_HOME'}
, 'hcathome' => $ENV{'HCAT_INSTALL_DIR'}
- , 'hcatalog.jar' =>
"$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
+ , 'hcatalog.jar' =>
"$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
#PIG
, 'testconfigpath' => "$ENV{PH_CLUSTER}"
@@ -72,7 +72,7 @@ $cfg = {
, 'pigpath' => "$ENV{PIG_HOME}"
, 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
- , 'additionaljars' =>
"$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.3.0-dev.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
+ , 'additionaljars' =>
"$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
#HADOOP
, 'hadoopHome' => "$ENV{HCAT_ROOT}/lib"
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
Sat Feb 25 01:58:06 2012
@@ -336,10 +336,15 @@ sub runHadoop
# Set HADOOP_CLASSPATH environment variable if provided
if (defined($testCmd->{'hadoop_classpath'})) {
- my $hadoop_classpath = $self->replaceParameters(
$testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log );
- my $cp = $testCmd->{'hcatalog.jar'};
+ #my $hadoop_classpath = $self->replaceParameters(
$testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log );
+ # TODO This is a total mess. Half the jars we need are specified in
hcatalog.jar, which is set in the default.conf file, and half are set in
hadoop_classpath, which
+ # has to be passed on the command line. And most of the jars are Hive
jars, thus they don't fit in the list of either hcatalog.jar or
hadoop_classpath. We need to
+ # make sense of this.
+ my $cp = $testCmd->{'hcatalog.jar'} . ":" .
$testCmd->{'hadoop_classpath'};
$cp =~ s/,/:/g;
$cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
+ # Add in the hcat config file
+ $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog";
$ENV{'HADOOP_CLASSPATH'} = $cp;
}
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf Sat Feb 25
01:58:06 2012
@@ -23,7 +23,7 @@ $cfg = {
,'hadoop' => q\
jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars
:HCAT_JAR: :THRIFTSERVER: studenttab10k :OUTPATH:
\,
- ,'sql' => q\select name, gpa from
studenttab10k;\
+ ,'sql' => q\select name, age from
studenttab10k;\
,'floatpostprocess' => 1
,'delimiter' => ' '
},
@@ -42,15 +42,46 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
{
'num' => 3
,'hcat_prep'=>q\drop table if exists
hadoop_checkin_3;
-create table hadoop_checkin_3 (name string, cnt int) STORED AS TEXTFILE;\
+create table hadoop_checkin_3 (age int, cnt int) STORED AS TEXTFILE;\
,'hadoop' => q\
jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.GroupByAge -libjars
:HCAT_JAR: :THRIFTSERVER: studenttab10k hadoop_checkin_3
\,
,'result_table' => 'hadoop_checkin_3'
,'sql' => q\select age, count(*) from
studenttab10k group by age;\
+ },{
+ # Read from a partitioned table
+ 'num' => 4
+ ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars
:HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH:
+\,
+ ,'sql' => q\select name, age from
studentparttab30k;\
,'floatpostprocess' => 1
,'delimiter' => ' '
- },
+ },{
+ # Write a single partition to a partitioned
table
+ 'num' => 5
+ ,'hcat_prep'=>q\drop table if exists
hadoop_checkin_5;
+create table hadoop_checkin_5 (name string, age int) partitioned by (ds
string) STORED AS TEXTFILE;\
+ ,'hadoop' => q?
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned
-libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k hadoop_checkin_5
ds=\"20110924\"
+?,
+ ,'result_table' => 'hadoop_checkin_5'
+ ,'sql' => q\select name, age, ds from
studentparttab30k where ds='20110924';\
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ }, {
+ # Write a multiple partitions to a
partitioned table
+ 'num' => 6
+ ,'hcat_prep'=>q\drop table if exists
hadoop_checkin_6;
+create table hadoop_checkin_6 (name string, age int) partitioned by (ds
string) STORED AS TEXTFILE;\
+ ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned
-libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k hadoop_checkin_6
+\,
+ ,'result_table' => 'hadoop_checkin_6'
+ ,'sql' => q\select name, age, ds from
studentparttab30k;\
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ }
],
}, # end g
{
@@ -122,10 +153,8 @@ create table hadoop_write_2(
d double,
m map<string, string>,
bb array<struct<a: int, b: string>>)
- STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat'
- INPUTDRIVER
'org.apache.hcatalog.pig.drivers.LoadFuncBasedInputDriver' OUTPUTDRIVER
'org.apache.hcatalog.pig.drivers.StoreFuncBasedOutputDriver'
- TBLPROPERTIES
('hcat.pig.loader'='org.apache.pig.builtin.JsonLoader',
'hcat.pig.storer'='org.apache.pig.builtin.JsonStorage', 'hcat.pig.loader.args'=
-'s:chararray, i:int, d:double, m:map[chararray], bb:{t:(a:int, b:chararray)}',
'hcat.pig.args.delimiter'=' ');\
+ row format serde 'org/apache/hcatalog/data/JsonSerDe'
+ stored as textfile;\
,'hadoop' => q\
jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteJson -libjars
:HCAT_JAR: :THRIFTSERVER: all100kjson hadoop_write_2
\,
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Sat Feb 25
01:58:06 2012
@@ -130,7 +130,7 @@ show databases;\
,'hcat' => q\
show tables;\,
,'rc' => 0
- ,'expected_out_regex' => 'studenttab10k'
+ ,'expected_out_regex' => 'hcat_createtable_1'
},
],
}, # end g
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Sat Feb 25
01:58:06 2012
@@ -77,7 +77,33 @@ store c into ':OUTPATH:';\,
,'sql' => [ 'select * from studenttab10k
where age<=40;', 'select * from studenttab10k where age>40;']
,'floatpostprocess' => 1
,'delimiter' => ' '
- },
+ },
+ {
+ 'num' => 6
+ ,'hcat_prep'=>q\drop table if exists
pig_checkin_6;
+create table pig_checkin_6 (name string, age int) partitioned by (ds string)
STORED AS TEXTFILE;\
+ ,'pig' => q\a = load 'studentparttab30k' using
org.apache.hcatalog.pig.HCatLoader();
+b = filter a by ds == '20110924';
+c = foreach b generate name, age;
+store c into 'pig_checkin_6' using
org.apache.hcatalog.pig.HCatStorer('ds=20110924');\,
+#dump a;\,
+ ,'result_table' => 'pig_checkin_6',
+ ,'sql' => "select name, age, ds from
studentparttab30k where ds='20110924';",
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ },
+ {
+ 'num' => 7
+ ,'hcat_prep'=>q\drop table if exists
pig_checkin_7;
+create table pig_checkin_7 (name string, age int) partitioned by (ds string)
STORED AS TEXTFILE;\
+ ,'pig' => q\a = load 'studentparttab30k' using
org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate name, age, ds;
+store b into 'pig_checkin_7' using org.apache.hcatalog.pig.HCatStorer();\,
+ ,'result_table' => 'pig_checkin_7',
+ ,'sql' => "select name, age, ds from
studentparttab30k;",
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' '
+ }
],
}, # end g
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
(original)
+++
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
Sat Feb 25 01:58:06 2012
@@ -55,8 +55,8 @@ public class ReadText extends Configured
public static class Map
extends Mapper<WritableComparable, HCatRecord, IntWritable, HCatRecord>{
- int t;
- int si;
+ byte t;
+ short si;
int i;
long b;
float f;
@@ -68,8 +68,8 @@ public class ReadText extends Configured
org.apache.hadoop.mapreduce.Mapper<WritableComparable,HCatRecord,
IntWritable,HCatRecord>.Context context)
throws IOException ,InterruptedException {
- t = (Integer)value.get(0);
- si = (Integer)value.get(1);
+ t = (Byte)value.get(0);
+ si = (Short)value.get(1);
i = (Integer)value.get(2);
b = (Long)value.get(3);
f = (Float)value.get(4);
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
(original)
+++
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
Sat Feb 25 01:58:06 2012
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
@@ -54,7 +54,7 @@ public class SimpleRead extends Configur
private static final String TAB = "\t";
public static class Map
- extends Mapper<WritableComparable, HCatRecord, Text, DoubleWritable>{
+ extends Mapper<WritableComparable, HCatRecord, Text, IntWritable>{
String name;
int age;
@@ -63,12 +63,13 @@ public class SimpleRead extends Configur
@Override
protected void map(WritableComparable key, HCatRecord value,
org.apache.hadoop.mapreduce.Mapper<WritableComparable,HCatRecord,
- Text,DoubleWritable>.Context context)
+ Text,IntWritable>.Context context)
throws IOException ,InterruptedException {
name = (String) value.get(0);
+System.out.println(name);
age = (Integer) value.get(1);
gpa = (Double) value.get(2);
- context.write(new Text(name), new DoubleWritable(gpa));
+ context.write(new Text(name), new IntWritable(age));
}
}
@@ -95,7 +96,7 @@ public class SimpleRead extends Configur
job.setJarByClass(SimpleRead.class);
job.setMapperClass(Map.class);
job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(DoubleWritable.class);
+ job.setOutputValueClass(IntWritable.class);
FileOutputFormat.setOutputPath(job, new Path(outputDir));
return (job.waitForCompletion(true) ? 0 : 1);
}
Modified:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
(original)
+++
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
Sat Feb 25 01:58:06 2012
@@ -53,8 +53,8 @@ public class WriteText extends Configure
public static class Map extends
Mapper<WritableComparable, HCatRecord, WritableComparable,
HCatRecord> {
- int t;
- int si;
+ byte t;
+ short si;
int i;
long b;
float f;
@@ -67,8 +67,8 @@ public class WriteText extends Configure
HCatRecord value,
org.apache.hadoop.mapreduce.Mapper<WritableComparable,
HCatRecord, WritableComparable, HCatRecord>.Context context)
throws IOException, InterruptedException {
- t = (Integer)value.get(0);
- si = (Integer)value.get(1);
+ t = (Byte)value.get(0);
+ si = (Short)value.get(1);
i = (Integer)value.get(2);
b = (Long)value.get(3);
f = (Float)value.get(4);
Added:
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java?rev=1293514&view=auto
==============================================================================
---
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
(added)
+++
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
Sat Feb 25 01:58:06 2012
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.utils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.data.DefaultHCatRecord;
+import org.apache.hcatalog.data.HCatRecord;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.hcatalog.mapreduce.HCatOutputFormat;
+import org.apache.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hcatalog.mapreduce.OutputJobInfo;
+
+/**
+ * This is a map reduce test for testing hcat writing to partitioned tables.
+ * table. It performs a group by on the first column and a SUM operation on the
+ * other columns. This is to simulate a typical operation in a map reduce
+ * program to test that hcat hands the right data to the map reduce program
+ *
+ * Usage: hadoop jar org.apache.hcatalog.utils.HBaseReadWrite -libjars
+ * <hcat_jar> * <serveruri> <input_tablename>
<output_tablename> [filter]
+ * If filter is given it will be provided as the partition to write to.
+ */
+public class WriteTextPartitioned extends Configured implements Tool {
+
+ static String filter = null;
+
+ public static class Map extends
+ Mapper<WritableComparable, HCatRecord, WritableComparable,
HCatRecord> {
+
+ @Override
+ protected void map(
+ WritableComparable key,
+ HCatRecord value,
+ org.apache.hadoop.mapreduce.Mapper<WritableComparable,
HCatRecord, WritableComparable, HCatRecord>.Context context)
+ throws IOException, InterruptedException {
+ String name = (String)value.get(0);
+ int age = (Integer)value.get(1);
+ String ds = (String)value.get(3);
+
+ HCatRecord record = (filter == null ? new DefaultHCatRecord(3) :
new DefaultHCatRecord(2));
+ record.set(0, name);
+ record.set(1, age);
+ if (filter == null) record.set(2, ds);
+
+ context.write(null, record);
+
+ }
+ }
+
+ public int run(String[] args) throws Exception {
+ Configuration conf = getConf();
+ args = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+ String serverUri = args[0];
+ String inputTableName = args[1];
+ String outputTableName = args[2];
+ if (args.length > 3) filter = args[3];
+ String dbName = null;
+
+ String principalID = System
+ .getProperty(HCatConstants.HCAT_METASTORE_PRINCIPAL);
+ if (principalID != null)
+ conf.set(HCatConstants.HCAT_METASTORE_PRINCIPAL, principalID);
+ Job job = new Job(conf, "WriteTextPartitioned");
+ HCatInputFormat.setInput(job, InputJobInfo.create(dbName,
+ inputTableName, filter, serverUri, principalID));
+ // initialize HCatOutputFormat
+
+ job.setInputFormatClass(HCatInputFormat.class);
+ job.setJarByClass(WriteTextPartitioned.class);
+ job.setMapperClass(Map.class);
+ job.setOutputKeyClass(WritableComparable.class);
+ job.setOutputValueClass(DefaultHCatRecord.class);
+ job.setNumReduceTasks(0);
+
+ java.util.Map<String, String> partitionVals = null;
+ if (filter != null) {
+ String[] s = filter.split("=");
+ String val = s[1].replace('"', ' ').trim();
+ partitionVals = new HashMap<String, String>(1);
+ partitionVals.put(s[0], val);
+ }
+ HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName,
+ outputTableName, partitionVals));
+ HCatSchema s = HCatInputFormat.getTableSchema(job);
+ // Build the schema for this table, which is slightly different than
the
+ // schema for the input table
+ List<HCatFieldSchema> fss = new ArrayList<HCatFieldSchema>(3);
+ fss.add(s.get(0));
+ fss.add(s.get(1));
+ fss.add(s.get(3));
+ HCatOutputFormat.setSchema(job, new HCatSchema(fss));
+ job.setOutputFormatClass(HCatOutputFormat.class);
+ return (job.waitForCompletion(true) ? 0 : 1);
+ }
+
+ public static void main(String[] args) throws Exception {
+ int exitCode = ToolRunner.run(new WriteTextPartitioned(), args);
+ System.exit(exitCode);
+ }
+}
