Author: szetszwo
Date: Thu Mar 15 18:29:13 2012
New Revision: 1301130
URL: http://svn.apache.org/viewvc?rev=1301130&view=rev
Log:
svn merge -c 1301127 from trunk for HDFS-3005.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
Propchange: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 15 18:29:13 2012
@@ -1,5 +1,5 @@
/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs:1227776-1294021
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182189,1182205,1182214,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189613,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1204709,1204825,1205146,1205260,1205626,1205697,1206178,1206786,1206830,1207585,
1207694,1208140,1208153,1208313,1210208,1210657,1210719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227887,1227964,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605,1234555,1235135,1235137,1235956,1236456,1238700,1238779,1238969,1239752,1240020,1240653,1240897,1240928,1241007,1241519,1242087,1242891,1243065,1243104,1243654,1243690,1244766,1245751,1245762,1292419,1292626,1293419,1293487,1295061,1295227,1295929,1297328,1298044,1298066,1298696,1298700,1299045,1299139,1299144,1299434,1299963,1300392
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182189,1182205,1182214,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189613,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1204709,1204825,1205146,1205260,1205626,1205697,1206178,1206786,1206830,1207585,
1207694,1208140,1208153,1208313,1210208,1210657,1210719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227887,1227964,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605,1234555,1235135,1235137,1235956,1236456,1238700,1238779,1238969,1239752,1240020,1240653,1240897,1240928,1241007,1241519,1242087,1242891,1243065,1243104,1243654,1243690,1244766,1245751,1245762,1292419,1292626,1293419,1293487,1295061,1295227,1295929,1297328,1298044,1298066,1298696,1298700,1299045,1299139,1299144,1299434,1299963,1300392,1301127
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1301130&r1=1301129&r2=1301130&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Thu Mar 15 18:29:13 2012
@@ -208,6 +208,8 @@ Release 0.23.3 - UNRELEASED
HDFS-3093. Fix bug where namenode -format interpreted the -force flag in
reverse. (todd)
+ HDFS-3005. FSVolume.decDfsUsed(..) should be synchronized. (szetszwo)
+
BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)
Propchange:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Mar 15 18:29:13 2012
@@ -1,5 +1,5 @@
/hadoop/common/branches/branch-0.23-PB/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1227776-1294021
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182189,1182205,1182214,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189613,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1205146,1205260,1205697,1206786,1206830,1207694,1208140,1208153,12
08313,1210208,1210657,1210719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227887,1227964,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605,1234555,1235135,1235137,1235956,1236456,1238700,1238779,1238969,1239752,1240020,1240653,1240897,1240928,1241007,1241519,1242087,1242891,1243065,1243104,1243654,1244766,1245751,1245762,1292419,1293419,1293487,1295061,1295227,1295929,1297328,1298044,1298696,1298700,1299045,1299139,1299144,1299434,1299963,1300392
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1161992,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1165826,1166402,1166466,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1171711,1172916,1173402,1173468,1173488,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1182189,1182205,1182214,1183081,1183098,1183175,1183554,1186508,1186896,1187140,1187505,1188282,1188286,1188300,1188436,1188487,1189028,1189355,1189360,1189546,1189613,1189901,1189932,1189982,1190077,1190127,1190620,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1196171,1197329,1198903,1199396,1200731,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204366,1204370,1204376,1204388,1204544,1204707,1205146,1205260,1205697,1206786,1206830,1207694,1208140,1208153,12
08313,1210208,1210657,1210719,1210746,1211206,1211249,1211769,1212021,1212062,1212073,1212084,1212299,1212606,1213040,1213143,1213537,1213586,1213592-1213593,1213808,1213813,1213954,1213985,1214027,1214033,1214046,1214102-1214103,1214128,1215364,1215366,1220315,1220510,1221106,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227887,1227964,1229347,1229877,1229897,1230398,1231569,1231572,1231627,1231640,1233584,1233605,1234555,1235135,1235137,1235956,1236456,1238700,1238779,1238969,1239752,1240020,1240653,1240897,1240928,1241007,1241519,1242087,1242891,1243065,1243104,1243654,1244766,1245751,1245762,1292419,1293419,1293487,1295061,1295227,1295929,1297328,1298044,1298696,1298700,1299045,1299139,1299144,1299434,1299963,1300392,1301127
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-1052/src/java:987665-1095512
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=1301130&r1=1301129&r2=1301130&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
Thu Mar 15 18:29:13 2012
@@ -95,15 +95,18 @@ class FSDataset implements FSDatasetInte
* A node type that can be built into a tree reflecting the
* hierarchy of blocks on the local disk.
*/
- private class FSDir {
+ private static class FSDir {
+ final int maxBlocksPerDir;
final File dir;
int numBlocks = 0;
FSDir children[];
int lastChildIdx = 0;
- private FSDir(File dir)
+ private FSDir(File dir, int maxBlocksPerDir)
throws IOException {
this.dir = dir;
+ this.maxBlocksPerDir = maxBlocksPerDir;
+
this.children = null;
if (!dir.exists()) {
if (!dir.mkdirs()) {
@@ -115,7 +118,7 @@ class FSDataset implements FSDatasetInte
List<FSDir> dirList = new ArrayList<FSDir>();
for (int idx = 0; idx < files.length; idx++) {
if (files[idx].isDirectory()) {
- dirList.add(new FSDir(files[idx]));
+ dirList.add(new FSDir(files[idx], maxBlocksPerDir));
} else if (Block.isBlockFilename(files[idx])) {
numBlocks++;
}
@@ -165,7 +168,8 @@ class FSDataset implements FSDatasetInte
if (children == null || children.length == 0) {
children = new FSDir[maxBlocksPerDir];
for (int idx = 0; idx < maxBlocksPerDir; idx++) {
- children[idx] = new FSDir(new File(dir,
DataStorage.BLOCK_SUBDIR_PREFIX+idx));
+ final File sub = new File(dir, DataStorage.BLOCK_SUBDIR_PREFIX+idx);
+ children[idx] = new FSDir(sub, maxBlocksPerDir);
}
}
@@ -297,8 +301,10 @@ class FSDataset implements FSDatasetInte
* A BlockPoolSlice represents a portion of a BlockPool stored on a volume.
* Taken together, all BlockPoolSlices sharing a block pool ID across a
* cluster represent a single block pool.
+ *
+ * This class is synchronized by {@link FSVolume}.
*/
- private class BlockPoolSlice {
+ private static class BlockPoolSlice {
private final String bpid;
private final FSVolume volume; // volume to which this BlockPool belongs to
private final File currentDir; // StorageDirectory/current/bpid/current
@@ -335,10 +341,16 @@ class FSDataset implements FSDatasetInte
FileUtil.fullyDelete(tmpDir);
}
this.rbwDir = new File(currentDir, DataStorage.STORAGE_DIR_RBW);
+ final boolean supportAppends = conf.getBoolean(
+ DFSConfigKeys.DFS_SUPPORT_APPEND_KEY,
+ DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT);
if (rbwDir.exists() && !supportAppends) {
FileUtil.fullyDelete(rbwDir);
}
- this.finalizedDir = new FSDir(finalizedDir);
+ final int maxBlocksPerDir = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_NUMBLOCKS_KEY,
+ DFSConfigKeys.DFS_DATANODE_NUMBLOCKS_DEFAULT);
+ this.finalizedDir = new FSDir(finalizedDir, maxBlocksPerDir);
if (!rbwDir.mkdirs()) { // create rbw directory if not exist
if (!rbwDir.isDirectory()) {
throw new IOException("Mkdirs failed to create " +
rbwDir.toString());
@@ -365,12 +377,12 @@ class FSDataset implements FSDatasetInte
return rbwDir;
}
+ /**
+ * This should be used only by {@link FSVolume#decDfsUsed(String, long)}
+ * and it will be synchronized there.
+ */
void decDfsUsed(long value) {
- // The caller to this method (BlockFileDeleteTask.run()) does
- // not have locked FSDataset.this yet.
- synchronized(FSDataset.this) {
- dfsUsage.decDfsUsed(value);
- }
+ dfsUsage.decDfsUsed(value);
}
long getDfsUsed() throws IOException {
@@ -530,14 +542,22 @@ class FSDataset implements FSDatasetInte
dfsUsage.shutdown();
}
}
-
- class FSVolume implements FSVolumeInterface {
+
+ /**
+ * The underlying volume used to store replica.
+ *
+ * It uses the {@link FSDataset} object for synchronization.
+ */
+ static class FSVolume implements FSVolumeInterface {
+ private final FSDataset dataset;
private final Map<String, BlockPoolSlice> map = new HashMap<String,
BlockPoolSlice>();
private final File currentDir; // <StorageDirectory>/current
private final DF usage;
private final long reserved;
- FSVolume(File currentDir, Configuration conf) throws IOException {
+ FSVolume(FSDataset dataset, File currentDir, Configuration conf
+ ) throws IOException {
+ this.dataset = dataset;
this.reserved = conf.getLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY,
DFSConfigKeys.DFS_DATANODE_DU_RESERVED_DEFAULT);
this.currentDir = currentDir;
@@ -555,9 +575,7 @@ class FSDataset implements FSDatasetInte
}
void decDfsUsed(String bpid, long value) {
- // The caller to this method (BlockFileDeleteTask.run()) does
- // not have locked FSDataset.this yet.
- synchronized(FSDataset.this) {
+ synchronized(dataset) {
BlockPoolSlice bp = map.get(bpid);
if (bp != null) {
bp.decDfsUsed(value);
@@ -566,11 +584,11 @@ class FSDataset implements FSDatasetInte
}
long getDfsUsed() throws IOException {
- // TODO valid synchronization
long dfsUsed = 0;
- Set<Entry<String, BlockPoolSlice>> set = map.entrySet();
- for (Entry<String, BlockPoolSlice> entry : set) {
- dfsUsed += entry.getValue().getDfsUsed();
+ synchronized(dataset) {
+ for(BlockPoolSlice s : map.values()) {
+ dfsUsed += s.getDfsUsed();
+ }
}
return dfsUsed;
}
@@ -630,11 +648,11 @@ class FSDataset implements FSDatasetInte
*/
@Override
public String[] getBlockPoolList() {
- synchronized(FSDataset.this) {
+ synchronized(dataset) {
return map.keySet().toArray(new String[map.keySet().size()]);
}
}
-
+
/**
* Temporary files. They get moved to the finalized block directory when
* the block is finalized.
@@ -658,14 +676,17 @@ class FSDataset implements FSDatasetInte
return bp.addBlock(b, f);
}
+ /**
+ * This should be used only by {@link FSVolumeSet#checkDirs()}
+ * and it will be synchronized there.
+ */
void checkDirs() throws DiskErrorException {
// TODO:FEDERATION valid synchronization
- Set<Entry<String, BlockPoolSlice>> set = map.entrySet();
- for (Entry<String, BlockPoolSlice> entry : set) {
- entry.getValue().checkDirs();
+ for(BlockPoolSlice s : map.values()) {
+ s.checkDirs();
}
}
-
+
void getVolumeMap(ReplicasMap volumeMap) throws IOException {
Set<Entry<String, BlockPoolSlice>> set = map.entrySet();
for (Entry<String, BlockPoolSlice> entry : set) {
@@ -877,31 +898,25 @@ class FSDataset implements FSDatasetInte
// Make a copy of volumes for performing modification
final List<FSVolume> volumeList = new ArrayList<FSVolume>(volumes);
- for (int idx = 0; idx < volumeList.size(); idx++) {
- FSVolume fsv = volumeList.get(idx);
+ for(Iterator<FSVolume> i = volumeList.iterator(); i.hasNext(); ) {
+ final FSVolume fsv = i.next();
try {
fsv.checkDirs();
} catch (DiskErrorException e) {
DataNode.LOG.warn("Removing failed volume " + fsv + ": ",e);
if (removedVols == null) {
- removedVols = new ArrayList<FSVolume>(1);
+ removedVols = new ArrayList<FSVolume>(2);
}
removedVols.add(fsv);
fsv.shutdown();
- volumeList.set(idx, null); // Remove the volume
+ i.remove(); // Remove the volume
numFailedVolumes++;
}
}
- // Remove null volumes from the volumes array
if (removedVols != null && removedVols.size() > 0) {
- final List<FSVolume> newVols = new ArrayList<FSVolume>();
- for (FSVolume vol : volumeList) {
- if (vol != null) {
- newVols.add(vol);
- }
- }
- volumes = Collections.unmodifiableList(newVols); // Replace volume list
+ // Replace volume list
+ volumes = Collections.unmodifiableList(volumeList);
DataNode.LOG.info("Completed FSVolumeSet.checkDirs. Removed "
+ removedVols.size() + " volumes. List of current volumes: "
+ this);
@@ -1048,7 +1063,6 @@ class FSDataset implements FSDatasetInte
private final DataNode datanode;
final FSVolumeSet volumes;
- private final int maxBlocksPerDir;
final ReplicasMap volumeMap;
final FSDatasetAsyncDiskService asyncDiskService;
private final int validVolsRequired;
@@ -1056,20 +1070,12 @@ class FSDataset implements FSDatasetInte
// Used for synchronizing access to usage stats
private final Object statsLock = new Object();
- final boolean supportAppends;
-
/**
* An FSDataset has a directory where it loads its data files.
*/
private FSDataset(DataNode datanode, DataStorage storage, Configuration conf
) throws IOException {
this.datanode = datanode;
- this.maxBlocksPerDir =
- conf.getInt(DFSConfigKeys.DFS_DATANODE_NUMBLOCKS_KEY,
- DFSConfigKeys.DFS_DATANODE_NUMBLOCKS_DEFAULT);
- this.supportAppends =
- conf.getBoolean(DFSConfigKeys.DFS_SUPPORT_APPEND_KEY,
- DFSConfigKeys.DFS_SUPPORT_APPEND_DEFAULT);
// The number of volumes required for operation is the total number
// of volumes minus the number of failed volumes we can tolerate.
final int volFailuresTolerated =
@@ -1098,7 +1104,7 @@ class FSDataset implements FSDatasetInte
storage.getNumStorageDirs());
for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) {
final File dir = storage.getStorageDir(idx).getCurrentDir();
- volArray.add(new FSVolume(dir, conf));
+ volArray.add(new FSVolume(this, dir, conf));
DataNode.LOG.info("FSDataset added volume - " + dir);
}
volumeMap = new ReplicasMap(this);
