Author: szetszwo
Date: Wed Feb 1 23:24:15 2012
New Revision: 1239398
URL: http://svn.apache.org/viewvc?rev=1239398&view=rev
Log:
Merge r1237584 through r1239397 from 0.23.
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/ (props
changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-auth/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
(contents, props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/docs/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/core/
(props changed)
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Feb 1 23:24:15 2012
@@ -0,0 +1 @@
+target
Propchange: hadoop/common/branches/branch-0.23-PB/hadoop-common-project/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,2 +1,2 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project:1227776-1239397
/hadoop/common/trunk/hadoop-common-project:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-auth/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,2 +1,2 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth:1227776-1239397
/hadoop/common/trunk/hadoop-common-project/hadoop-auth:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1179869,1182189,1182205,1182214,1189357,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,2 +1,2 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common:1227776-1237583
-/hadoop/common/trunk/hadoop-common-project/hadoop-common:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166009,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231
569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common:1227776-1239397
+/hadoop/common/trunk/hadoop-common-project/hadoop-common:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166009,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231
569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1239398&r1=1239397&r2=1239398&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Feb 1 23:24:15 2012
@@ -150,9 +150,17 @@ Release 0.23.1 - Unreleased
HADOOP-8002. SecurityUtil acquired token message should be a debug rather
than info.
(Arpit Gupta via mahadev)
+ HADOOP-8009. Create hadoop-client and hadoop-minicluster artifacts for
downstream
+ projects. (tucu)
+
+ HADOOP-7470. Move up to Jackson 1.8.8. (Enis Soztutar via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
+
+ HADOOP-8006 TestFSInputChecker is failing in trunk.
+ (Daryn Sharp via bobby)
HADOOP-7998. CheckFileSystem does not correctly honor setVerifyChecksum
(Daryn Sharp via bobby)
@@ -247,6 +255,9 @@ Release 0.23.1 - Unreleased
HADOOP-8000. fetchdt command not available in bin/hadoop.
(Arpit Gupta via mahadev)
+ HADOOP-7999. "hadoop archive" fails with ClassNotFoundException.
+ (Jason Lowe via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,6 +1,6 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt:1227776-1239397
/hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164771,1166009,1166402,1167318,1167383,1169986,1170046,1170379,1170459,1171221,1171297,1171894,1171909,1172186,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204376,1204388,1205260,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350-1226351,1227091,1227165,1227423,1227861,1227964,
1229347,1230398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164771,1166009,1166402,1167318,1167383,1169986,1170046,1170379,1170459,1171221,1171297,1171894,1171909,1172186,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1179869,1182189,1182205,1182214,1182641,1183132,1189357,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204376,1204388,1205260,1206830,1207694,1208153,1208313,1209246,1210208,1210319-1210320,1212004,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213598,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350-1226351,1227091,1227165,1227423,1227861,1227964,
1229347,1230398,1231569,1231572,1231627,1231640,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456,1239386
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh?rev=1239398&r1=1239397&r2=1239398&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
Wed Feb 1 23:24:15 2012
@@ -20,6 +20,16 @@
# Resolve links ($0 may be a softlink) and convert a relative path
# to an absolute path. NB: The -P option requires bash built-ins
# or POSIX:2001 compliant cd and pwd.
+
+# HADOOP_CLASSPATH Extra Java CLASSPATH entries.
+#
+# HADOOP_USER_CLASSPATH_FIRST When defined, the HADOOP_CLASSPATH is
+# added in the beginning of the global
+# classpath. Can be defined, for example,
+# by doing
+# export HADOOP_USER_CLASSPATH_FIRST=true
+#
+
this="${BASH_SOURCE-$0}"
common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
script="$(basename -- "$this")"
@@ -153,6 +163,10 @@ fi
# CLASSPATH initially contains $HADOOP_CONF_DIR
CLASSPATH="${HADOOP_CONF_DIR}"
+if [ "$HADOOP_USER_CLASSPATH_FIRST" != "" ] && [ "$HADOOP_CLASSPATH" != "" ] ;
then
+ CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH}
+fi
+
# so that filenames w/ spaces are handled correctly in loops below
IFS=
@@ -174,7 +188,7 @@ fi
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR'/*'
# add user-specified CLASSPATH last
-if [ "$HADOOP_CLASSPATH" != "" ]; then
+if [ "$HADOOP_USER_CLASSPATH_FIRST" = "" ] && [ "$HADOOP_CLASSPATH" != "" ];
then
CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH}
fi
@@ -214,6 +228,9 @@ if [ -d "${HADOOP_PREFIX}/build/native"
fi
fi
+# setup a default TOOL_PATH
+TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"
+
# cygwin path translation
if $cygwin; then
JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,3 +1,3 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs:1227776-1239397
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1183132,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1209246,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java:1227776-1239397
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1164771,1166402,1167318,1167383,1170379,1170459,1171221,1171297,1172916,1173402,1176550,1176986,1177002,1177035,1177399,1177487,1177531,1177859,1177864,1178639,1179512,1182189,1182205,1182214,1182641,1183132,1189932,1189982,1190109,1190611,1195575,1195760,1196113,1196129,1197885,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204363,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1210208,1210319-1210320,1212004,1212021,1212062,1212073,1212084,1212615,1213537,1213586,1213592-1213593,1213619,1213954,1214046,1214114,1214499,1215358,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227861,1227964,1229347,1230398,1231569,1231572,1231627,123164
0,1231732,1233605,1234555,1235135,1235137,1235956,1236444,1236456
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1239398&r1=1239397&r2=1239398&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
Wed Feb 1 23:24:15 2012
@@ -119,7 +119,6 @@ public abstract class ChecksumFileSystem
private static final int HEADER_LENGTH = 8;
private int bytesPerSum = 1;
- private long fileLen = -1L;
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
throws IOException {
@@ -244,6 +243,24 @@ public abstract class ChecksumFileSystem
}
return nread;
}
+ }
+
+ private static class FSDataBoundedInputStream extends FSDataInputStream {
+ private FileSystem fs;
+ private Path file;
+ private long fileLen = -1L;
+
+ FSDataBoundedInputStream(FileSystem fs, Path file, InputStream in)
+ throws IOException {
+ super(in);
+ this.fs = fs;
+ this.file = file;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
/* Return the file length */
private long getFileLength() throws IOException {
@@ -304,9 +321,16 @@ public abstract class ChecksumFileSystem
*/
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- return verifyChecksum
- ? new FSDataInputStream(new ChecksumFSInputChecker(this, f, bufferSize))
- : getRawFileSystem().open(f, bufferSize);
+ FileSystem fs;
+ InputStream in;
+ if (verifyChecksum) {
+ fs = this;
+ in = new ChecksumFSInputChecker(this, f, bufferSize);
+ } else {
+ fs = getRawFileSystem();
+ in = fs.open(f, bufferSize);
+ }
+ return new FSDataBoundedInputStream(fs, f, in);
}
/** {@inheritDoc} */
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 1 23:24:15 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/core:1227776-1237583
+/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/core:1227776-1239397
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1161777,1161781,1162008,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163456,1163465,1163490,1163768,1163852,1163858,1164255,1164301,1164339,1166009,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1183132,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
