Author: olga
Date: Wed Sep 16 18:53:22 2009
New Revision: 815926
URL: http://svn.apache.org/viewvc?rev=815926&view=rev
Log:
PIG-660: Integration with Hadoop 20 (sms via olgan)
Added:
hadoop/pig/trunk/lib/hadoop20.jar (with props)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/bin/pig
hadoop/pig/trunk/build.xml
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Sep 16 18:53:22 2009
@@ -28,6 +28,7 @@
BUG FIXES
+PIG-660: Integration with Hadoop 20 (sms via olgan)
PIG-962: Skewed join creates 3 map reduce jobs (sriranjan via olgan)
Release 0.4.0 - Unreleased
Modified: hadoop/pig/trunk/bin/pig
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/bin/pig?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/bin/pig (original)
+++ hadoop/pig/trunk/bin/pig Wed Sep 16 18:53:22 2009
@@ -34,7 +34,7 @@
#
# PIG_ROOT_LOGGER The root appender. Default is INFO,console
#
-# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 18
(0.18).
+# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 20
(0.20).
cygwin=false
case "`uname`" in
@@ -136,7 +136,7 @@
done
# Set the version for Hadoop, default to 17
-PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-18}"
+PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}"
# add libs to CLASSPATH. There can be more than one version of the hadoop
# libraries in the lib dir, so don't blindly add them all. Only add the one
# that matche PIG_HADOOP_VERSION.
Modified: hadoop/pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Wed Sep 16 18:53:22 2009
@@ -47,14 +47,14 @@
<!-- property name="build.encoding" value="ISO-8859-1" / -->
<property name="build.encoding" value="UTF8" />
<!-- TODO with only one version of hadoop in the lib folder we do not need
that anymore -->
- <property name="hadoop.jarfile" value="hadoop18.jar" />
+ <property name="hadoop.jarfile" value="hadoop20.jar" />
<property name="hbase.jarfile" value="hbase-0.18.1.jar" />
<property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" />
<!-- javac properties -->
<property name="javac.debug" value="on" />
<property name="javac.optimize" value="on" />
- <property name="javac.deprecation" value="on" />
+ <property name="javac.deprecation" value="off" />
<property name="javac.version" value="1.5" />
<property name="javac.args" value="" />
<!-- default warnings option -->
@@ -74,7 +74,7 @@
<property name="test.build.dir" value="${build.dir}/test" />
<property name="test.build.classes" value="${test.build.dir}/classes" />
<property name="test.log.dir" value="${test.build.dir}/logs" />
- <property name="test.timeout" value="900000" />
+ <property name="test.timeout" value="2700000" />
<property name="test.junit.output.format" value="plain" />
<!-- test configuration, use ${user.home}/build.properties to configure
values -->
@@ -452,6 +452,7 @@
<!-- Excluded under Windows.-->
<exclude name="**/TestHBaseStorage.java" if="isWindows" />
<!-- Excluced because we don't want to run them -->
+ <exclude name="**/TestHBaseStorage.java" />
<exclude name="**/PigExecTestCase.java" />
<exclude name="**/TypeCheckingTestUtil.java" />
<exclude name="**/TypeGraphPrinter.java" />
Added: hadoop/pig/trunk/lib/hadoop20.jar
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/lib/hadoop20.jar?rev=815926&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/pig/trunk/lib/hadoop20.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
Wed Sep 16 18:53:22 2009
@@ -23,7 +23,6 @@
import java.util.Enumeration;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
import java.util.Properties;
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
Wed Sep 16 18:53:22 2009
@@ -30,7 +30,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.dfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigException;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Wed Sep 16 18:53:22 2009
@@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
@@ -1955,10 +1956,10 @@
ExecutionEngine eng = pigContext.getExecutionEngine();
if(eng instanceof HExecutionEngine){
try {
- val = Math.round(0.9f *
((HExecutionEngine)eng).getJobClient().getDefaultReduces());
+ val =
((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks();
if(val<=0)
val = 1;
- } catch (IOException e) {
+ } catch (Exception e) {
int errCode = 6015;
String msg = "Problem getting the default number of
reduces from the Job Client.";
throw new MRCompilerException(msg, errCode,
PigException.REMOTE_ENVIRONMENT, e);
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/io/NullableBytesWritable.java Wed
Sep 16 18:53:22 2009
@@ -39,6 +39,6 @@
public Object getValueAsPigType() {
BytesWritable bw = (BytesWritable)mValue;
- return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize());
+ return isNull() ? null : new DataByteArray(bw.getBytes(), 0,
bw.getLength());
}
}
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Wed Sep 16
18:53:22 2009
@@ -154,12 +154,12 @@
if (counters!=null)
{
Counters.Group taskgroup =
counters.getGroup("org.apache.hadoop.mapred.Task$Counter");
- Counters.Group hdfsgroup =
counters.getGroup("org.apache.hadoop.mapred.Task$FileSystemCounter");
+ Counters.Group hdfsgroup =
counters.getGroup("FileSystemCounters");
jobStats.put("PIG_STATS_MAP_INPUT_RECORDS",
(Long.valueOf(taskgroup.getCounterForName("MAP_INPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_MAP_OUTPUT_RECORDS",
(Long.valueOf(taskgroup.getCounterForName("MAP_OUTPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_REDUCE_INPUT_RECORDS",
(Long.valueOf(taskgroup.getCounterForName("REDUCE_INPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_REDUCE_OUTPUT_RECORDS",
(Long.valueOf(taskgroup.getCounterForName("REDUCE_OUTPUT_RECORDS").getCounter())).toString());
- jobStats.put("PIG_STATS_BYTES_WRITTEN",
(Long.valueOf(hdfsgroup.getCounterForName("HDFS_WRITE").getCounter())).toString());
+ jobStats.put("PIG_STATS_BYTES_WRITTEN",
(Long.valueOf(hdfsgroup.getCounterForName("HDFS_BYTES_WRITTEN").getCounter())).toString());
}
else
{
Modified: hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java?rev=815926&r1=815925&r2=815926&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/MiniCluster.java Wed Sep 16
18:53:22 2009
@@ -20,7 +20,7 @@
import java.io.*;
import java.util.Properties;
-import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.conf.Configuration;
@@ -68,7 +68,7 @@
m_conf.setInt("mapred.submit.replication", 2);
m_conf.set("dfs.datanode.address", "0.0.0.0:0");
m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
- m_conf.write(new FileOutputStream(conf_file));
+ m_conf.writeXml(new FileOutputStream(conf_file));
// Set the system properties needed by Pig
System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
