Author: vinodkv
Date: Tue Sep 3 18:30:05 2013
New Revision: 1519787
URL: http://svn.apache.org/r1519787
Log:
Forwarding YARN-321 branch to latest branch-2.
Added:
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
- copied unchanged from r1519783,
hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
- copied unchanged from r1519783,
hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
Modified:
hadoop/common/branches/YARN-321/ (props changed)
hadoop/common/branches/YARN-321/BUILDING.txt
hadoop/common/branches/YARN-321/dev-support/test-patch.sh
hadoop/common/branches/YARN-321/hadoop-dist/pom.xml
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
hadoop/common/branches/YARN-321/hadoop-project/ (props changed)
hadoop/common/branches/YARN-321/hadoop-project/pom.xml (contents, props
changed)
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-datajoin/ (props
changed)
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/ (props
changed)
Propchange: hadoop/common/branches/YARN-321/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2:r1513206-1519783
Modified: hadoop/common/branches/YARN-321/BUILDING.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/BUILDING.txt?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/BUILDING.txt (original)
+++ hadoop/common/branches/YARN-321/BUILDING.txt Tue Sep 3 18:30:05 2013
@@ -7,7 +7,7 @@ Requirements:
* JDK 1.6
* Maven 3.0
* Findbugs 1.3.9 (if running findbugs)
-* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
+* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code)
* Internet connection for first build (to fetch all Maven and Hadoop
dependencies)
@@ -100,6 +100,16 @@ time out after a while, using the Maven
to update SNAPSHOTs from external repos.
----------------------------------------------------------------------------------
+Protocol Buffer compiler
+
+The version of Protocol Buffer compiler, protoc, must match the version of the
+protobuf JAR.
+
+If you have multiple versions of protoc in your system, you can set in your
+build shell the HADOOP_PROTOC_PATH environment variable to point to the one
you
+want to use for the Hadoop build. If you don't define this environment
variable,
+protoc is looked up in the PATH.
+----------------------------------------------------------------------------------
Importing projects to eclipse
When you import the project to eclipse, install hadoop-maven-plugins at first.
Modified: hadoop/common/branches/YARN-321/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/dev-support/test-patch.sh?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/YARN-321/dev-support/test-patch.sh Tue Sep 3
18:30:05 2013
@@ -395,6 +395,9 @@ checkJavadocWarnings () {
echo ""
echo "There appear to be $javadocWarnings javadoc warnings generated by the
patched build."
+ #There are 11 warnings that are caused by things that are caused by using
sun internal APIs.
+ #There are 2 warnings that are caused by the Apache DS Dn class used in
MiniKdc.
+ OK_JAVADOC_WARNINGS=13;
### if current warnings greater than OK_JAVADOC_WARNINGS
if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
Modified: hadoop/common/branches/YARN-321/hadoop-dist/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-dist/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-dist/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-dist/pom.xml Tue Sep 3 18:30:05 2013
@@ -115,8 +115,10 @@
run mkdir hadoop-${project.version}
run cd hadoop-${project.version}
run cp -r
$ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/*
.
+ run cp -r
$ROOT/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${project.version}/* .
run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/*
.
+ run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version}/*
.
run cp -r
$ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
run cp -r
$ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
run cp -r
$ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/*
.
Modified:
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
Tue Sep 3 18:30:05 2013
@@ -45,19 +45,45 @@ public class ProtocMojo extends Abstract
@Parameter(required=true)
private FileSet source;
- @Parameter(defaultValue="protoc")
+ @Parameter
private String protocCommand;
+ @Parameter(required=true)
+ private String protocVersion;
public void execute() throws MojoExecutionException {
try {
+ if (protocCommand == null || protocCommand.trim().isEmpty()) {
+ protocCommand = "protoc";
+ }
+ List<String> command = new ArrayList<String>();
+ command.add(protocCommand);
+ command.add("--version");
+ Exec exec = new Exec(this);
+ List<String> out = new ArrayList<String>();
+ if (exec.run(command, out) == 127) {
+ getLog().error("protoc, not found at: " + protocCommand);
+ throw new MojoExecutionException("protoc failure");
+ } else {
+ if (out.isEmpty()) {
+ getLog().error("stdout: " + out);
+ throw new MojoExecutionException(
+ "'protoc --version' did not return a version");
+ } else {
+ if (!out.get(0).endsWith(protocVersion)) {
+ throw new MojoExecutionException(
+ "protoc version is '" + out.get(0) + "', expected version is
'"
+ + protocVersion + "'");
+ }
+ }
+ }
if (!output.mkdirs()) {
if (!output.exists()) {
throw new MojoExecutionException("Could not create directory: " +
output);
}
}
- List<String> command = new ArrayList<String>();
+ command = new ArrayList<String>();
command.add(protocCommand);
command.add("--java_out=" + output.getCanonicalPath());
if (imports != null) {
@@ -68,8 +94,8 @@ public class ProtocMojo extends Abstract
for (File f : FileSetUtils.convertFileSetToFiles(source)) {
command.add(f.getCanonicalPath());
}
- Exec exec = new Exec(this);
- List<String> out = new ArrayList<String>();
+ exec = new Exec(this);
+ out = new ArrayList<String>();
if (exec.run(command, out) != 0) {
getLog().error("protoc compiler error");
for (String s : out) {
Modified:
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
Tue Sep 3 18:30:05 2013
@@ -63,11 +63,10 @@ public class Exec {
for (String s : stdErr.getOutput()) {
mojo.getLog().debug(s);
}
- } else {
- stdOut.join();
- stdErr.join();
- output.addAll(stdOut.getOutput());
}
+ stdOut.join();
+ stdErr.join();
+ output.addAll(stdOut.getOutput());
} catch (Exception ex) {
mojo.getLog().warn(command + " failed: " + ex.toString());
}
Propchange: hadoop/common/branches/YARN-321/hadoop-project/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-project:r1513206-1519783
Modified: hadoop/common/branches/YARN-321/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-project/pom.xml Tue Sep 3 18:30:05
2013
@@ -58,6 +58,11 @@
<!-- at different nesting levels in the source tree may need to override.
-->
<hadoop.common.build.dir>${basedir}/../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
<java.security.egd>file:///dev/urandom</java.security.egd>
+
+ <!-- ProtocolBuffer version, used to verify the protoc version and -->
+ <!-- define the protobuf JAR version -->
+ <protobuf.version>2.5.0</protobuf.version>
+ <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
</properties>
<dependencyManagement>
@@ -289,6 +294,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>11.0.2</version>
@@ -608,7 +619,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>2.4.0a</version>
+ <version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>commons-daemon</groupId>
@@ -652,6 +663,13 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.2</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.bookkeeper</groupId>
<artifactId>bookkeeper-server</artifactId>
<version>4.0.0</version>
Propchange: hadoop/common/branches/YARN-321/hadoop-project/pom.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/branch-2/hadoop-project/pom.xml:r1513206-1519783
Propchange: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-datajoin/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Tue Sep 3 18:30:05 2013
@@ -1 +1,4 @@
target
+.classpath
+.project
+.settings
Propchange: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Tue Sep 3 18:30:05 2013
@@ -1 +1,4 @@
target
+.classpath
+.project
+.settings
