Author: gkesavan
Date: Fri Aug 3 19:00:15 2012
New Revision: 1369164
URL: http://svn.apache.org/viewvc?rev=1369164&view=rev
Log:
merge -r1358479:HEAD http://svn.apache.org/repos/asf/hadoop/common/trunk
Added:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/
- copied from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamBaseRecordReader.java
- copied unchanged from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamBaseRecordReader.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java
- copied unchanged from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamXmlRecordReader.java
- copied unchanged from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamXmlRecordReader.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/
- copied from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java
- copied unchanged from r1369130,
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java
Removed:
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-raid-dist.xml
Modified:
hadoop/common/branches/branch-trunk-win/ (props changed)
hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamBaseRecordReader.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamXmlRecordReader.java
Propchange: hadoop/common/branches/branch-trunk-win/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1358480-1369130
Modified: hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh Fri Aug
3 19:00:15 2012
@@ -921,8 +921,9 @@ fi
checkTests
(( RESULT = RESULT + $? ))
applyPatch
-(( RESULT = RESULT + $? ))
-if [[ $RESULT != 0 ]] ; then
+APPLY_PATCH_RET=$?
+(( RESULT = RESULT + $APPLY_PATCH_RET ))
+if [[ $APPLY_PATCH_RET != 0 ]] ; then
submitJiraComment 1
cleanupAndExit 1
fi
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
Fri Aug 3 19:00:15 2012
@@ -111,9 +111,9 @@
<outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory>
</fileSet>
<fileSet>
- <directory>${basedir}/src/main/native</directory>
+ <directory>${basedir}/src/main/native/libhdfs</directory>
<includes>
- <include>*.h</include>
+ <include>hdfs.h</include>
</includes>
<outputDirectory>/include</outputDirectory>
</fileSet>
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
Fri Aug 3 19:00:15 2012
@@ -126,9 +126,6 @@
</fileSets>
<moduleSets>
<moduleSet>
- <excludes>
- <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
- </excludes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}</outputDirectory>
<includeDependencies>false</includeDependencies>
@@ -138,6 +135,7 @@
<moduleSet>
<includes>
<include>org.apache.hadoop:hadoop-mapreduce-client-jobclient</include>
+ <include>org.apache.hadoop:hadoop-yarn-server-tests</include>
</includes>
<binaries>
<attachmentClassifier>tests</attachmentClassifier>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-dist/pom.xml Fri Aug 3
19:00:15 2012
@@ -52,11 +52,6 @@
<artifactId>hadoop-yarn-api</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-raid</artifactId>
- <scope>provided</scope>
- </dependency>
</dependencies>
<build>
@@ -125,7 +120,6 @@
run cp -r
$ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/*
.
run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/*
.
- run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs-raid/target/hadoop-hdfs-raid-${project.version}/*
.
run cp -r
$ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
run cp -r
$ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/*
.
echo
Modified: hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml Fri Aug 3
19:00:15 2012
@@ -197,6 +197,12 @@
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-applications-distributedshell</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
@@ -257,11 +263,6 @@
<artifactId>hadoop-client</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-raid</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -648,7 +649,12 @@
<version>4.0.0</version>
<scope>compile</scope>
</dependency>
-
+ <dependency>
+ <groupId>com.cenqua.clover</groupId>
+ <artifactId>clover</artifactId>
+ <!-- Use the version needed by maven-clover-plugin -->
+ <version>3.0.2</version>
+ </dependency>
</dependencies>
</dependencyManagement>
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
Fri Aug 3 19:00:15 2012
@@ -66,6 +66,7 @@
<item name="Writing Yarn Applications"
href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
<item name="Capacity Scheduler"
href="hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html"/>
<item name="Web Application Proxy"
href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
+ <item name="Encrypted Shuffle"
href="hadoop-yarn/hadoop-yarn-site/EncryptedShuffle.html"/>
<item name="Yarn Commands"
href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
</menu>
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamBaseRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamBaseRecordReader.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamBaseRecordReader.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamBaseRecordReader.java
Fri Aug 3 19:00:15 2012
@@ -126,7 +126,7 @@ public abstract class StreamBaseRecordRe
}
String unqualSplit = split_.getPath().getName() + ":" +
split_.getStart() + "+" + split_.getLength();
- String status = "HSTR " + StreamUtil.HOST + " " + numRec_ + ". pos=" + pos
+ " " + unqualSplit
+ String status = "HSTR " + StreamUtil.getHost() + " " + numRec_ + ". pos="
+ pos + " " + unqualSplit
+ " Processing record=" + recStr;
status += " " + splitName_;
return status;
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Fri Aug 3 19:00:15 2012
@@ -958,7 +958,6 @@ public class StreamJob implements Tool {
if (!b)
fail(LINK_URI);
}
- DistributedCache.createSymlink(jobConf_);
// set the jobconf for the caching parameters
if (cacheArchives != null)
DistributedCache.setCacheArchives(archiveURIs, jobConf_);
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
Fri Aug 3 19:00:15 2012
@@ -168,12 +168,17 @@ public class StreamUtil {
}
static private Environment env;
- static String HOST;
+ private static String host;
+ public static String getHost(){
+ return host;
+ }
+
+
static {
try {
env = new Environment();
- HOST = env.getHost();
+ host = env.getHost();
} catch (IOException io) {
io.printStackTrace();
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamXmlRecordReader.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamXmlRecordReader.java?rev=1369164&r1=1369163&r2=1369164&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamXmlRecordReader.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamXmlRecordReader.java
Fri Aug 3 19:00:15 2012
@@ -64,7 +64,7 @@ public class StreamXmlRecordReader exten
init();
}
- public void init() throws IOException {
+ public final void init() throws IOException {
LOG.info("StreamBaseRecordReader.init: " + " start_=" + start_ + " end_="
+ end_ + " length_="
+ length_ + " start_ > in_.getPos() =" + (start_ > in_.getPos())
+ " " + start_ + " > "
+ in_.getPos());
@@ -185,14 +185,14 @@ public class StreamXmlRecordReader exten
}
// states
- final static int CDATA_IN = 10;
- final static int CDATA_OUT = 11;
- final static int CDATA_UNK = 12;
- final static int RECORD_ACCEPT = 13;
+ private final static int CDATA_IN = 10;
+ private final static int CDATA_OUT = 11;
+ private final static int CDATA_UNK = 12;
+ private final static int RECORD_ACCEPT = 13;
// inputs
- final static int CDATA_BEGIN = 20;
- final static int CDATA_END = 21;
- final static int RECORD_MAYBE = 22;
+ private final static int CDATA_BEGIN = 20;
+ private final static int CDATA_END = 21;
+ private final static int RECORD_MAYBE = 22;
/* also updates firstMatchStart_;*/
int nextState(int state, int input, int bufPos) {
@@ -293,7 +293,7 @@ public class StreamXmlRecordReader exten
BufferedInputStream bin_; // Wrap FSDataInputStream for efficient backward
seeks
long pos_; // Keep track on position with respect encapsulated
FSDataInputStream
- final static int NA = -1;
+ private final static int NA = -1;
int firstMatchStart_ = 0; // candidate record boundary. Might just be CDATA.
int firstMatchEnd_ = 0;
