Author: szetszwo
Date: Sat Jan 21 19:50:30 2012
New Revision: 1234406
URL: http://svn.apache.org/viewvc?rev=1234406&view=rev
Log:
Merge r1233741 through r1234387 from 0.23.
Removed:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182205,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1204709,1204825,1205146,1205260,1205626,1205697,1206178,1206786,1206830,1207585,1207694,1208140,1208153,
1208313,1210208,1210657,1210719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1227887,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
Sat Jan 21 19:50:30 2012
@@ -1,3 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<FindBugsFilter>
<Match>
<Class
name="org.apache.hadoop.lib.service.instrumentation.InstrumentationService" />
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Sat Jan 21 19:50:30 2012
@@ -199,6 +199,8 @@ Release 0.23.1 - UNRELEASED
HDFS-2803. Add logging to LeaseRenewer for better lease expiration
debugging.
(Jimmy Xiang via todd)
+ HDFS-2817. Combine the two TestSafeMode test suites. (todd)
+
OPTIMIZATIONS
HDFS-2130. Switch default checksum to CRC32C. (todd)
@@ -266,6 +268,14 @@ Release 0.23.1 - UNRELEASED
HDFS-2790. FSNamesystem.setTimes throws exception with wrong
configuration name in the message. (Arpit Gupta via eli)
+ HDFS-2810. Leases not getting renewed properly by clients (todd)
+
+ HDFS-2751. Datanode may incorrectly drop OS cache behind reads
+ even for short reads. (todd)
+
+ HDFS-2816. Fix missing license header in httpfs findbugsExcludeFile.xml.
+ (hitesh via tucu)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182205,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1205146,1205260,1205697,1206786,1206830,1207694,1208140,1208153,1208313,1210208,1210657,12
10719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1227887,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
Sat Jan 21 19:50:30 2012
@@ -373,11 +373,17 @@ public class DFSClient implements java.i
return clientRunning;
}
- /** Renew leases */
- void renewLease() throws IOException {
+ /**
+ * Renew leases.
+ * @return true if lease was renewed. May return false if this
+ * client has been closed or has no files open.
+ **/
+ boolean renewLease() throws IOException {
if (clientRunning && !isFilesBeingWrittenEmpty()) {
namenode.renewLease(clientName);
+ return true;
}
+ return false;
}
/**
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
Sat Jan 21 19:50:30 2012
@@ -67,7 +67,7 @@ import org.apache.hadoop.util.StringUtil
* </p>
*/
class LeaseRenewer {
- private static final Log LOG = LogFactory.getLog(LeaseRenewer.class);
+ static final Log LOG = LogFactory.getLog(LeaseRenewer.class);
static final long LEASE_RENEWER_GRACE_DEFAULT = 60*1000L;
static final long LEASE_RENEWER_SLEEP_DEFAULT = 1000L;
@@ -407,7 +407,13 @@ class LeaseRenewer {
final DFSClient c = copies.get(i);
//skip if current client name is the same as the previous name.
if (!c.getClientName().equals(previousName)) {
- c.renewLease();
+ if (!c.renewLease()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Did not renew lease for client " +
+ c);
+ }
+ continue;
+ }
previousName = c.getClientName();
if (LOG.isDebugEnabled()) {
LOG.debug("Lease renewed for client " + previousName);
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
Sat Jan 21 19:50:30 2012
@@ -315,7 +315,7 @@ class BlockSender implements java.io.Clo
* close opened files.
*/
public void close() throws IOException {
- if (blockInFd != null && shouldDropCacheBehindRead) {
+ if (blockInFd != null && shouldDropCacheBehindRead && isLongRead()) {
// drop the last few MB of the file from cache
try {
NativeIO.posixFadviseIfPossible(
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640,1233605
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640,1233605
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
/hadoop/core/trunk/src/webapps/datanode:776175-784663
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640,1233605
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640,1233605
/hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
/hadoop/core/trunk/src/webapps/secondary:776175-784663
Propchange:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Jan 21 19:50:30 2012
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1227776-1233740
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1227776-1234387
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182205,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225192,1225456,1225489,1225591,1226211,1226239,1227091,1227165,1227423,1229347,1230398,1231569,1231572,1231627,1231640,1233605
/hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
/hadoop/core/trunk/src/test/hdfs:776175-785643
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRenewer.java
Sat Jan 21 19:50:30 2012
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.hdfs;
+import static org.junit.Assert.*;
+
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -29,6 +32,8 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import com.google.common.base.Supplier;
+
public class TestLeaseRenewer {
private String FAKE_AUTHORITY="hdfs://nn1/";
private UserGroupInformation FAKE_UGI_A =
@@ -46,19 +51,24 @@ public class TestLeaseRenewer {
@Before
public void setupMocksAndRenewer() throws IOException {
- MOCK_DFSCLIENT = Mockito.mock(DFSClient.class);
- Mockito.doReturn(true)
- .when(MOCK_DFSCLIENT).isClientRunning();
- Mockito.doReturn((int)FAST_GRACE_PERIOD)
- .when(MOCK_DFSCLIENT).getHdfsTimeout();
- Mockito.doReturn("myclient")
- .when(MOCK_DFSCLIENT).getClientName();
+ MOCK_DFSCLIENT = createMockClient();
renewer = LeaseRenewer.getInstance(
FAKE_AUTHORITY, FAKE_UGI_A, MOCK_DFSCLIENT);
renewer.setGraceSleepPeriod(FAST_GRACE_PERIOD);
}
+ private DFSClient createMockClient() {
+ DFSClient mock = Mockito.mock(DFSClient.class);
+ Mockito.doReturn(true)
+ .when(mock).isClientRunning();
+ Mockito.doReturn((int)FAST_GRACE_PERIOD)
+ .when(mock).getHdfsTimeout();
+ Mockito.doReturn("myclient")
+ .when(mock).getClientName();
+ return mock;
+ }
+
@Test
public void testInstanceSharing() throws IOException {
// Two lease renewers with the same UGI should return
@@ -93,11 +103,11 @@ public class TestLeaseRenewer {
public void testRenewal() throws Exception {
// Keep track of how many times the lease gets renewed
final AtomicInteger leaseRenewalCount = new AtomicInteger();
- Mockito.doAnswer(new Answer<Void>() {
+ Mockito.doAnswer(new Answer<Boolean>() {
@Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
leaseRenewalCount.incrementAndGet();
- return null;
+ return true;
}
}).when(MOCK_DFSCLIENT).renewLease();
@@ -120,6 +130,57 @@ public class TestLeaseRenewer {
renewer.closeFile(filePath, MOCK_DFSCLIENT);
}
+ /**
+ * Regression test for HDFS-2810. In this bug, the LeaseRenewer has handles
+ * to several DFSClients with the same name, the first of which has no files
+ * open. Previously, this was causing the lease to not get renewed.
+ */
+ @Test
+ public void testManyDfsClientsWhereSomeNotOpen() throws Exception {
+ // First DFSClient has no files open so doesn't renew leases.
+ final DFSClient mockClient1 = createMockClient();
+ Mockito.doReturn(false).when(mockClient1).renewLease();
+ assertSame(renewer, LeaseRenewer.getInstance(
+ FAKE_AUTHORITY, FAKE_UGI_A, mockClient1));
+
+ // Set up a file so that we start renewing our lease.
+ DFSOutputStream mockStream1 = Mockito.mock(DFSOutputStream.class);
+ String filePath = "/foo";
+ renewer.put(filePath, mockStream1, mockClient1);
+
+ // Second DFSClient does renew lease
+ final DFSClient mockClient2 = createMockClient();
+ Mockito.doReturn(true).when(mockClient2).renewLease();
+ assertSame(renewer, LeaseRenewer.getInstance(
+ FAKE_AUTHORITY, FAKE_UGI_A, mockClient2));
+
+ // Set up a file so that we start renewing our lease.
+ DFSOutputStream mockStream2 = Mockito.mock(DFSOutputStream.class);
+ renewer.put(filePath, mockStream2, mockClient2);
+
+
+ // Wait for lease to get renewed
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ Mockito.verify(mockClient1, Mockito.atLeastOnce()).renewLease();
+ Mockito.verify(mockClient2, Mockito.atLeastOnce()).renewLease();
+ return true;
+ } catch (AssertionError err) {
+ LeaseRenewer.LOG.warn("Not yet satisfied", err);
+ return false;
+ } catch (IOException e) {
+ // should not throw!
+ throw new RuntimeException(e);
+ }
+ }
+ }, 100, 10000);
+
+ renewer.closeFile(filePath, mockClient1);
+ renewer.closeFile(filePath, mockClient2);
+ }
+
@Test
public void testThreadName() throws Exception {
DFSOutputStream mockStream = Mockito.mock(DFSOutputStream.class);
Modified:
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1234406&r1=1234405&r2=1234406&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
(original)
+++
hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
Sat Jan 21 19:50:30 2012
@@ -113,6 +113,21 @@ public class TestSafeMode {
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE));
}
+ /**
+ * Test that, if there are no blocks in the filesystem,
+ * the NameNode doesn't enter the "safemode extension" period.
+ */
+ @Test(timeout=45000)
+ public void testNoExtensionIfNoBlocks() throws IOException {
+ cluster.getConfiguration(0).setInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 60000);
+ cluster.restartNameNode();
+ // Even though we have safemode extension set high, we should immediately
+ // exit safemode on startup because there are no blocks in the namespace.
+ String status = cluster.getNameNode().getNamesystem().getSafemode();
+ assertEquals("", status);
+ }
+
public interface FSRun {
public abstract void run(FileSystem fs) throws IOException;
}
@@ -193,5 +208,37 @@ public class TestSafeMode {
assertFalse("Could not leave SM",
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE));
}
-
+
+ /**
+ * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min
+ * is set to a number greater than the number of live datanodes.
+ */
+ @Test
+ public void testDatanodeThreshold() throws IOException {
+ cluster.shutdown();
+ Configuration conf = cluster.getConfiguration(0);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1);
+
+ cluster.restartNameNode();
+ fs = (DistributedFileSystem)cluster.getFileSystem();
+
+ String tipMsg = cluster.getNamesystem().getSafemode();
+ assertTrue("Safemode tip message looks right: " + tipMsg,
+ tipMsg.contains("The number of live datanodes 0 needs an
additional " +
+ "2 live datanodes to reach the minimum number
1. " +
+ "Safe mode will be turned off automatically."));
+
+ // Start a datanode
+ cluster.startDataNodes(conf, 1, true, null, null);
+
+ // Wait long enough for safemode check to refire
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ignored) {}
+
+ // We now should be out of safe mode.
+ assertEquals("", cluster.getNamesystem().getSafemode());
+ }
+
}
\ No newline at end of file
