Author: todd
Date: Fri Aug 26 22:46:17 2011
New Revision: 1162279
URL: http://svn.apache.org/viewvc?rev=1162279&view=rev
Log:
Merge trunk into HA branch
Added:
hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
- copied unchanged from r1162221,
hadoop/common/trunk/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
hadoop/common/branches/HDFS-1623/hadoop-project-dist/
- copied from r1162221, hadoop/common/trunk/hadoop-project-dist/
hadoop/common/branches/HDFS-1623/hadoop-project-dist/README.txt
- copied unchanged from r1162221,
hadoop/common/trunk/hadoop-project-dist/README.txt
hadoop/common/branches/HDFS-1623/hadoop-project-dist/pom.xml
- copied unchanged from r1162221,
hadoop/common/trunk/hadoop-project-dist/pom.xml
Modified:
hadoop/common/branches/HDFS-1623/ (props changed)
hadoop/common/branches/HDFS-1623/BUILDING.txt
hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh
hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh
hadoop/common/branches/HDFS-1623/hadoop-annotations/
hadoop/common/branches/HDFS-1623/hadoop-assemblies/pom.xml
hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-bintar.xml
hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-tar.xml
hadoop/common/branches/HDFS-1623/hadoop-common/
hadoop/common/branches/HDFS-1623/hadoop-hdfs/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce/
hadoop/common/branches/HDFS-1623/hadoop-project-distro/
hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml
hadoop/common/branches/HDFS-1623/pom.xml
Propchange: hadoop/common/branches/HDFS-1623/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Aug 26 22:46:17 2011
@@ -1 +1 @@
-/hadoop/common/trunk:1152502-1159756
+/hadoop/common/trunk:1152502-1162221
Modified: hadoop/common/branches/HDFS-1623/BUILDING.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/BUILDING.txt?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/BUILDING.txt (original)
+++ hadoop/common/branches/HDFS-1623/BUILDING.txt Fri Aug 26 22:46:17 2011
@@ -15,12 +15,13 @@ Requirements:
----------------------------------------------------------------------------------
Maven modules:
- hadoop (Main Hadoop project)
- - hadoop-project (Parent POM for all Hadoop Maven modules.
)
- (All plugins & dependencies versions are defined
here.)
- - hadoop-annotations (Generates the Hadoop doclet used to generated
the Javadocs)
- - hadoop-common (Hadoop Common)
- - hadoop-hdfs (Hadoop HDFS)
+ hadoop (Main Hadoop project)
+ - hadoop-project (Parent POM for all Hadoop Maven modules.
)
+ (All plugins & dependencies versions are
defined here.)
+ - hadoop-project-dist (Parent POM for modules that generate
distributions.)
+ - hadoop-annotations (Generates the Hadoop doclet used to generated
the Javadocs)
+ - hadoop-common (Hadoop Common)
+ - hadoop-hdfs (Hadoop HDFS)
----------------------------------------------------------------------------------
Where to run Maven from?
@@ -43,15 +44,16 @@ Maven build goals:
* Run clover : mvn test -Pclover
[-DcloverLicenseLocation=${user.name}/.clover.license]
* Run Rat : mvn apache-rat:check
* Build javadocs : mvn javadoc:javadoc
- * Build TAR : mvn package
[-Ptar][-Pbintar][-Pdocs][-Psrc][-Pnative]
+ * Build distribution : mvn package
[-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar]
Build options:
* Use -Pnative to compile/bundle native code
* Use -Dsnappy.prefix=(/usr/local) & -Dbundle.snappy=(false) to compile
Snappy JNI bindings and to bundle Snappy SO files
- * Use -Pdocs to generate & bundle the documentation in the TAR (using -Ptar)
- * Use -Psrc to bundle the source in the TAR (using -Ptar)
+ * Use -Pdocs to generate & bundle the documentation in the distribution
(using -Pdist)
+ * Use -Psrc to bundle the source in the distribution (using -Pdist)
+ * Use -Dtar to create a TAR with the distribution (using -Pdist)
Tests options:
Modified: hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh (original)
+++ hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh Fri Aug
26 22:46:17 2011
@@ -56,18 +56,18 @@ fi
PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
# if we are at the project root then nothing more to do
-if [[ -d hadoop-common ]]; then
+if [[ -d hadoop-common-project ]]; then
echo Looks like this is being run at project root
-# if all of the lines start with hadoop-common/, hadoop-hdfs/, or mapreduce/,
this is
+# if all of the lines start with hadoop-common/, hadoop-hdfs/, or
hadoop-mapreduce/, this is
# relative to the hadoop root instead of the subproject root, so we need
# to chop off another layer
-elif [[ "$PREFIX_DIRS" =~ ^(hadoop-hdfs|hadoop-common|mapreduce)$ ]]; then
+elif [[ "$PREFIX_DIRS" =~
^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
echo Looks like this is relative to project root. Increasing PLEVEL
PLEVEL=$[$PLEVEL + 1]
-elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common\|hadoop-hdfs\|mapreduce'
; then
+elif ! echo "$PREFIX_DIRS" | grep -vxq
'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
echo Looks like this is a cross-subproject patch. Try applying from the
project root
exit 1
fi
Modified: hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh Fri Aug 26
22:46:17 2011
@@ -370,8 +370,8 @@ checkJavadocWarnings () {
if [ -d hadoop-project ]; then
(cd hadoop-project; $MVN install)
fi
- if [ -d hadoop-annotations ]; then
- (cd hadoop-annotations; $MVN install)
+ if [ -d hadoop-common-project/hadoop-annotations ]; then
+ (cd hadoop-common-project/hadoop-annotations; $MVN install)
fi
$MVN clean compile javadoc:javadoc -DskipTests -Pdocs
-D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt |
$AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total
+= 1} END {print total}'`
@@ -446,7 +446,7 @@ checkReleaseAuditWarnings () {
echo ""
echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
- find . -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
+ find $BASEDIR -name rat.txt | xargs cat >
$PATCH_DIR/patchReleaseAuditWarnings.txt
### Compare trunk and patch release audit warning numbers
if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
@@ -520,8 +520,8 @@ checkFindbugsWarnings () {
echo "======================================================================"
echo ""
echo ""
- echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess"
- $MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess
+ echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess"
+ $MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess <
/dev/null
if [ $? != 0 ] ; then
JIRA_COMMENT="$JIRA_COMMENT
@@ -536,6 +536,7 @@ checkFindbugsWarnings () {
relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
if [ ! $relative_file == "target/findbugsXml.xml" ]; then
module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing
path
+ module_suffix=`basename ${module_suffix}`
fi
cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
@@ -549,7 +550,7 @@ checkFindbugsWarnings () {
$FINDBUGS_HOME/bin/convertXmlToText -html \
$PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
$PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
- JIRA_COMMENT_FOOTER="Findbugs warnings:
$BUILD_URL/artifact/trunk/target/newPatchFindbugsWarnings${module_suffix}.html
+ JIRA_COMMENT_FOOTER="Findbugs warnings:
$BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
$JIRA_COMMENT_FOOTER"
done
@@ -567,43 +568,84 @@ $JIRA_COMMENT_FOOTER"
}
###############################################################################
-### Run the test-core target
-runCoreTests () {
+### Run the tests
+runTests () {
echo ""
echo ""
echo "======================================================================"
echo "======================================================================"
- echo " Running core tests."
+ echo " Running tests."
echo "======================================================================"
echo "======================================================================"
echo ""
echo ""
- ### Kill any rogue build processes from the last attempt
- $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' |
/usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
- PreTestTarget=""
- if [[ $defect == MAPREDUCE-* ]] ; then
- PreTestTarget="create-c++-configure"
- fi
-
- echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
- $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- ### Find and format names of failed tests
- failed_tests=`grep -l -E "<failure|<error"
$WORKSPACE/trunk/target/hadoop-common/surefire-reports/*.xml | sed -e
"s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
+ failed_tests=""
+ modules=$(findModules)
+ for module in $modules;
+ do
+ pushd $module
+ echo " Running tests in $module"
+ ### Kill any rogue build processes from the last attempt
+ $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' |
/usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+
+ echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+ $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
+ if [[ $? != 0 ]] ; then
+ ### Find and format names of failed tests
+ module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E
"<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|
|g" | sed -e "s|\.xml||g"`
+ failed_tests="${failed_tests}
+${module_failed_tests}"
+ fi
+ popd
+ done
+ echo $failed_tests
+
+ if [[ -n "$failed_tests" ]] ; then
+
JIRA_COMMENT="$JIRA_COMMENT
- -1 core tests. The patch failed these core unit tests:
+ -1 core tests. The patch failed these unit tests:
$failed_tests"
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
- +1 core tests. The patch passed core unit tests."
+ +1 core tests. The patch passed unit tests in $modules."
return 0
}
###############################################################################
+### Find the modules changed by the patch
+
+findModules () {
+ # Come up with a list of changed files into $TMP
+ TMP=/tmp/tmp.paths.$$
+ $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null |
sort | uniq > $TMP
+
+ # if all of the lines start with a/ or b/, then this is a git patch that
+ # was generated without --no-prefix
+ if ! $GREP -qv '^a/\|^b/' $TMP ; then
+ sed -i -e 's,^[ab]/,,' $TMP
+ fi
+
+ PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
+
+ # if all of the lines start with hadoop-common-project/,
hadoop-hdfs-project/, or hadoop-mapreduce-project/, this is
+ # relative to the hadoop root instead of the subproject root
+ if [[ "$PREFIX_DIRS" =~
^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
+ echo $PREFIX_DIRS
+ return 0
+ elif ! echo "$PREFIX_DIRS" | grep -vxq
'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
+ echo $PREFIX_DIRS
+ return 0
+ fi
+
+ # No modules found. Running from current directory.
+ echo .
+}
+
+###############################################################################
### Run the test-contrib target
runContribTests () {
echo ""
@@ -658,6 +700,7 @@ checkInjectSystemFaults () {
#echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess=
-Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes
-Dforrest.home=$FORREST_HOME inject-system-faults"
#$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess=
-Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes
-Dforrest.home=$FORREST_HOME inject-system-faults
echo "NOP"
+ return 0
if [[ $? != 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
@@ -779,7 +822,7 @@ checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
### Do not call these when run by a developer
if [[ $JENKINS == "true" ]] ; then
- runCoreTests
+ runTests
(( RESULT = RESULT + $? ))
runContribTests
(( RESULT = RESULT + $? ))
Modified: hadoop/common/branches/HDFS-1623/hadoop-assemblies/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-assemblies/pom.xml?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-assemblies/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-assemblies/pom.xml Fri Aug 26
22:46:17 2011
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>0.23.0-SNAPSHOT</version>
+ <version>0.24.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-assemblies</artifactId>
- <version>0.23.0-SNAPSHOT</version>
+ <version>0.24.0-SNAPSHOT</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
Modified: hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-project/pom.xml Fri Aug 26 22:46:17
2011
@@ -16,7 +16,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>0.23.0-SNAPSHOT</version>
+ <version>0.24.0-SNAPSHOT</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
@@ -104,6 +104,11 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-alfredo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -473,6 +478,16 @@
<artifactId>jspc-maven-plugin</artifactId>
<version>2.0-alpha-3</version>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>2.1.1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
</plugins>
</pluginManagement>
Modified: hadoop/common/branches/HDFS-1623/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/pom.xml?rev=1162279&r1=1162278&r2=1162279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/pom.xml Fri Aug 26 22:46:17 2011
@@ -16,7 +16,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>0.23.0-SNAPSHOT</version>
+ <version>0.24.0-SNAPSHOT</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
@@ -35,11 +35,11 @@
<modules>
<module>hadoop-project</module>
- <module>hadoop-project-distro</module>
+ <module>hadoop-project-dist</module>
<module>hadoop-assemblies</module>
- <module>hadoop-annotations</module>
- <module>hadoop-common</module>
- <module>hadoop-hdfs</module>
+ <module>hadoop-common-project</module>
+ <module>hadoop-hdfs-project</module>
+ <module>hadoop-mapreduce-project</module>
</modules>
<build>
