Author: olga
Date: Wed Sep 16 18:53:22 2009
New Revision: 815926

URL: http://svn.apache.org/viewvc?rev=815926&view=rev
Log:
PIG-660: Integration with Hadoop 20 (sms via olgan)

Added:
    hadoop/pig/trunk/lib/hadoop20.jar   (with props)
Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/bin/pig
    hadoop/pig/trunk/build.xml
    
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
    
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
    
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
    hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java
    hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
    hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Sep 16 18:53:22 2009
@@ -28,6 +28,7 @@
 
 BUG FIXES
 
+PIG-660: Integration with Hadoop 20 (sms via olgan)
 PIG-962: Skewed join creates 3 map reduce jobs (sriranjan via olgan)
 
 Release 0.4.0 - Unreleased

Modified: hadoop/pig/trunk/bin/pig
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/bin/pig?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/bin/pig (original)
+++ hadoop/pig/trunk/bin/pig Wed Sep 16 18:53:22 2009
@@ -34,7 +34,7 @@
 #
 #     PIG_ROOT_LOGGER The root appender. Default is INFO,console
 #
-#     PIG_HADOOP_VERSION Version of hadoop to run with.    Default is 18 
(0.18).
+#     PIG_HADOOP_VERSION Version of hadoop to run with.    Default is 20 
(0.20).
 
 cygwin=false
 case "`uname`" in
@@ -136,7 +136,7 @@
 done
 
 # Set the version for Hadoop, default to 17
-PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-18}"
+PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}"
 # add libs to CLASSPATH.    There can be more than one version of the hadoop
 # libraries in the lib dir, so don't blindly add them all.    Only add the one
 # that matche PIG_HADOOP_VERSION.

Modified: hadoop/pig/trunk/build.xml
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Wed Sep 16 18:53:22 2009
@@ -47,14 +47,14 @@
     <!-- property name="build.encoding" value="ISO-8859-1" / -->
     <property name="build.encoding" value="UTF8" />
     <!-- TODO with only one version of hadoop in the lib folder we do not need 
that anymore -->
-    <property name="hadoop.jarfile" value="hadoop18.jar" />
+    <property name="hadoop.jarfile" value="hadoop20.jar" />
     <property name="hbase.jarfile" value="hbase-0.18.1.jar" />
     <property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" />
 
     <!-- javac properties -->
     <property name="javac.debug" value="on" />
     <property name="javac.optimize" value="on" />
-    <property name="javac.deprecation" value="on" />
+    <property name="javac.deprecation" value="off" />
     <property name="javac.version" value="1.5" />
     <property name="javac.args" value="" />
     <!-- default warnings option -->
@@ -74,7 +74,7 @@
     <property name="test.build.dir" value="${build.dir}/test" />
     <property name="test.build.classes" value="${test.build.dir}/classes" />
     <property name="test.log.dir" value="${test.build.dir}/logs" />
-    <property name="test.timeout" value="900000" />
+    <property name="test.timeout" value="2700000" />
     <property name="test.junit.output.format" value="plain" />
 
     <!-- test configuration, use ${user.home}/build.properties to configure 
values  -->
@@ -452,6 +452,7 @@
                     <!-- Excluded under Windows.-->
                     <exclude name="**/TestHBaseStorage.java" if="isWindows" />
                     <!-- Excluced because we don't want to run them -->
+                    <exclude name="**/TestHBaseStorage.java" />
                     <exclude name="**/PigExecTestCase.java" />
                     <exclude name="**/TypeCheckingTestUtil.java" />
                     <exclude name="**/TypeGraphPrinter.java" />

Added: hadoop/pig/trunk/lib/hadoop20.jar
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/lib/hadoop20.jar?rev=815926&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/pig/trunk/lib/hadoop20.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
 (original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
 Wed Sep 16 18:53:22 2009
@@ -23,7 +23,6 @@
 import java.util.Enumeration;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
 
 import java.util.Properties;
 

Modified: 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
 (original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
 Wed Sep 16 18:53:22 2009
@@ -30,7 +30,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.dfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.PigException;
 import org.apache.pig.backend.datastorage.ContainerDescriptor;

Modified: 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
 (original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
 Wed Sep 16 18:53:22 2009
@@ -31,6 +31,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.PigException;
 import org.apache.pig.PigWarning;
@@ -1955,10 +1956,10 @@
             ExecutionEngine eng = pigContext.getExecutionEngine();
             if(eng instanceof HExecutionEngine){
                 try {
-                    val = Math.round(0.9f * 
((HExecutionEngine)eng).getJobClient().getDefaultReduces());
+                    val = 
((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks();
                     if(val<=0)
                         val = 1;
-                } catch (IOException e) {
+                } catch (Exception e) {
                     int errCode = 6015;
                     String msg = "Problem getting the default number of 
reduces from the Job Client.";
                     throw new MRCompilerException(msg, errCode, 
PigException.REMOTE_ENVIRONMENT, e);

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java 
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java Wed 
Sep 16 18:53:22 2009
@@ -39,6 +39,6 @@
 
     public Object getValueAsPigType() {
         BytesWritable bw = (BytesWritable)mValue;
-        return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize());
+        return isNull() ? null : new DataByteArray(bw.getBytes(), 0, 
bw.getLength());
     }
 }

Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Wed Sep 16 
18:53:22 2009
@@ -154,12 +154,12 @@
                     if (counters!=null)
                     {
                         Counters.Group taskgroup = 
counters.getGroup("org.apache.hadoop.mapred.Task$Counter");
-                        Counters.Group hdfsgroup = 
counters.getGroup("org.apache.hadoop.mapred.Task$FileSystemCounter");
+                        Counters.Group hdfsgroup = 
counters.getGroup("FileSystemCounters");
                         jobStats.put("PIG_STATS_MAP_INPUT_RECORDS", 
(Long.valueOf(taskgroup.getCounterForName("MAP_INPUT_RECORDS").getCounter())).toString());
                         jobStats.put("PIG_STATS_MAP_OUTPUT_RECORDS", 
(Long.valueOf(taskgroup.getCounterForName("MAP_OUTPUT_RECORDS").getCounter())).toString());
                         jobStats.put("PIG_STATS_REDUCE_INPUT_RECORDS", 
(Long.valueOf(taskgroup.getCounterForName("REDUCE_INPUT_RECORDS").getCounter())).toString());
                         jobStats.put("PIG_STATS_REDUCE_OUTPUT_RECORDS", 
(Long.valueOf(taskgroup.getCounterForName("REDUCE_OUTPUT_RECORDS").getCounter())).toString());
-                        jobStats.put("PIG_STATS_BYTES_WRITTEN", 
(Long.valueOf(hdfsgroup.getCounterForName("HDFS_WRITE").getCounter())).toString());
+                        jobStats.put("PIG_STATS_BYTES_WRITTEN", 
(Long.valueOf(hdfsgroup.getCounterForName("HDFS_BYTES_WRITTEN").getCounter())).toString());
                     }
                     else
                     {

Modified: hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java Wed Sep 16 
18:53:22 2009
@@ -20,7 +20,7 @@
 import java.io.*;
 import java.util.Properties;
 
-import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapred.MiniMRCluster;
 
 import org.apache.hadoop.conf.Configuration;
@@ -68,7 +68,7 @@
             m_conf.setInt("mapred.submit.replication", 2);
             m_conf.set("dfs.datanode.address", "0.0.0.0:0");
             m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
-            m_conf.write(new FileOutputStream(conf_file));
+            m_conf.writeXml(new FileOutputStream(conf_file));
             
             // Set the system properties needed by Pig
             System.setProperty("cluster", m_conf.get("mapred.job.tracker"));


Reply via email to