Author: vinodkv Date: Tue Sep 3 18:30:05 2013 New Revision: 1519787 URL: http://svn.apache.org/r1519787 Log: Forwarding YARN-321 branch to latest branch-2.
Added: hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml - copied unchanged from r1519783, hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml - copied unchanged from r1519783, hadoop/common/branches/branch-2/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml Modified: hadoop/common/branches/YARN-321/ (props changed) hadoop/common/branches/YARN-321/BUILDING.txt hadoop/common/branches/YARN-321/dev-support/test-patch.sh hadoop/common/branches/YARN-321/hadoop-dist/pom.xml hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java hadoop/common/branches/YARN-321/hadoop-project/ (props changed) hadoop/common/branches/YARN-321/hadoop-project/pom.xml (contents, props changed) hadoop/common/branches/YARN-321/hadoop-tools/hadoop-datajoin/ (props changed) hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/ (props changed) Propchange: hadoop/common/branches/YARN-321/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2:r1513206-1519783 Modified: hadoop/common/branches/YARN-321/BUILDING.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/BUILDING.txt?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/BUILDING.txt (original) +++ hadoop/common/branches/YARN-321/BUILDING.txt Tue Sep 3 18:30:05 2013 @@ -7,7 +7,7 @@ Requirements: * JDK 1.6 * Maven 3.0 * Findbugs 1.3.9 (if running findbugs) -* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS) +* ProtocolBuffer 2.5.0 * CMake 2.6 or newer (if compiling native code) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) @@ -100,6 +100,16 @@ time out after a while, using the Maven to update SNAPSHOTs from external repos. ---------------------------------------------------------------------------------- +Protocol Buffer compiler + +The version of Protocol Buffer compiler, protoc, must match the version of the +protobuf JAR. + +If you have multiple versions of protoc in your system, you can set in your +build shell the HADOOP_PROTOC_PATH environment variable to point to the one you +want to use for the Hadoop build. If you don't define this environment variable, +protoc is looked up in the PATH. +---------------------------------------------------------------------------------- Importing projects to eclipse When you import the project to eclipse, install hadoop-maven-plugins at first. Modified: hadoop/common/branches/YARN-321/dev-support/test-patch.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/dev-support/test-patch.sh?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/dev-support/test-patch.sh (original) +++ hadoop/common/branches/YARN-321/dev-support/test-patch.sh Tue Sep 3 18:30:05 2013 @@ -395,6 +395,9 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." + #There are 11 warnings that are caused by things that are caused by using sun internal APIs. + #There are 2 warnings that are caused by the Apache DS Dn class used in MiniKdc. + OK_JAVADOC_WARNINGS=13; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT Modified: hadoop/common/branches/YARN-321/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-dist/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-dist/pom.xml Tue Sep 3 18:30:05 2013 @@ -115,8 +115,10 @@ run mkdir hadoop-${project.version} run cd hadoop-${project.version} run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* . + run cp -r $ROOT/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* . + run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version}/* . run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* . run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* . run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* . Modified: hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java (original) +++ hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java Tue Sep 3 18:30:05 2013 @@ -45,19 +45,45 @@ public class ProtocMojo extends Abstract @Parameter(required=true) private FileSet source; - @Parameter(defaultValue="protoc") + @Parameter private String protocCommand; + @Parameter(required=true) + private String protocVersion; public void execute() throws MojoExecutionException { try { + if (protocCommand == null || protocCommand.trim().isEmpty()) { + protocCommand = "protoc"; + } + List<String> command = new ArrayList<String>(); + command.add(protocCommand); + command.add("--version"); + Exec exec = new Exec(this); + List<String> out = new ArrayList<String>(); + if (exec.run(command, out) == 127) { + getLog().error("protoc, not found at: " + protocCommand); + throw new MojoExecutionException("protoc failure"); + } else { + if (out.isEmpty()) { + getLog().error("stdout: " + out); + throw new MojoExecutionException( + "'protoc --version' did not return a version"); + } else { + if (!out.get(0).endsWith(protocVersion)) { + throw new MojoExecutionException( + "protoc version is '" + out.get(0) + "', expected version is '" + + protocVersion + "'"); + } + } + } if (!output.mkdirs()) { if (!output.exists()) { throw new MojoExecutionException("Could not create directory: " + output); } } - List<String> command = new ArrayList<String>(); + command = new ArrayList<String>(); command.add(protocCommand); command.add("--java_out=" + output.getCanonicalPath()); if (imports != null) { @@ -68,8 +94,8 @@ public class ProtocMojo extends Abstract for (File f : FileSetUtils.convertFileSetToFiles(source)) { command.add(f.getCanonicalPath()); } - Exec exec = new Exec(this); - List<String> out = new ArrayList<String>(); + exec = new Exec(this); + out = new ArrayList<String>(); if (exec.run(command, out) != 0) { getLog().error("protoc compiler error"); for (String s : out) { Modified: hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java (original) +++ hadoop/common/branches/YARN-321/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java Tue Sep 3 18:30:05 2013 @@ -63,11 +63,10 @@ public class Exec { for (String s : stdErr.getOutput()) { mojo.getLog().debug(s); } - } else { - stdOut.join(); - stdErr.join(); - output.addAll(stdOut.getOutput()); } + stdOut.join(); + stdErr.join(); + output.addAll(stdOut.getOutput()); } catch (Exception ex) { mojo.getLog().warn(command + " failed: " + ex.toString()); } Propchange: hadoop/common/branches/YARN-321/hadoop-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-project:r1513206-1519783 Modified: hadoop/common/branches/YARN-321/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-project/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-project/pom.xml Tue Sep 3 18:30:05 2013 @@ -58,6 +58,11 @@ <!-- at different nesting levels in the source tree may need to override. --> <hadoop.common.build.dir>${basedir}/../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir> <java.security.egd>file:///dev/urandom</java.security.egd> + + <!-- ProtocolBuffer version, used to verify the protoc version and --> + <!-- define the protobuf JAR version --> + <protobuf.version>2.5.0</protobuf.version> + <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path> </properties> <dependencyManagement> @@ -289,6 +294,12 @@ </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minikdc</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>11.0.2</version> @@ -608,7 +619,7 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>2.4.0a</version> + <version>${protobuf.version}</version> </dependency> <dependency> <groupId>commons-daemon</groupId> @@ -652,6 +663,13 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.4.2</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.bookkeeper</groupId> <artifactId>bookkeeper-server</artifactId> <version>4.0.0</version> Propchange: hadoop/common/branches/YARN-321/hadoop-project/pom.xml ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-project/pom.xml:r1513206-1519783 Propchange: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-datajoin/ ------------------------------------------------------------------------------ --- svn:ignore (original) +++ svn:ignore Tue Sep 3 18:30:05 2013 @@ -1 +1,4 @@ target +.classpath +.project +.settings Propchange: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/ ------------------------------------------------------------------------------ --- svn:ignore (original) +++ svn:ignore Tue Sep 3 18:30:05 2013 @@ -1 +1,4 @@ target +.classpath +.project +.settings