Author: szetszwo
Date: Fri Nov 7 18:24:09 2008
New Revision: 712344
URL: http://svn.apache.org/viewvc?rev=712344&view=rev
Log:
HADOOP-4583. Several code optimizations in HDFS. (Suresh Srinivas via szetszwo)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=712344&r1=712343&r2=712344&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Nov 7 18:24:09 2008
@@ -70,6 +70,9 @@
HADOOP-4453. Improve ssl configuration and handling in HsftpFileSystem,
particularly when used with DistCp. (Kan Zhang via cdouglas)
+ HADOOP-4583. Several code optimizations in HDFS. (Suresh Srinivas via
+ szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -964,9 +967,6 @@
HADOOP-4288. Fixes a NPE problem in CapacityScheduler.
(Amar Kamat via ddas)
- HADOOP-3883. Limit namenode to assign at most one generation stamp for
- a particular block within a short period. (szetszwo)
-
HADOOP-4014. Create hard links with 'fsutil hardlink' on Windows. (shv)
HADOOP-4393. Merged org.apache.hadoop.fs.permission.AccessControlException
@@ -1085,6 +1085,9 @@
HADOOP-4610. Always calculate mis-replicated blocks when safe-mode is
turned off. (shv)
+ HADOOP-3883. Limit namenode to assign at most one generation stamp for
+ a particular block within a short period. (szetszwo)
+
Release 0.18.2 - 2008-11-03
BUG FIXES
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=712344&r1=712343&r2=712344&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Fri Nov 7 18:24:09 2008
@@ -449,12 +449,13 @@
for (int idx = 0; idx < errorStreams.size(); idx++) {
EditLogOutputStream eStream = errorStreams.get(idx);
int j = 0;
- for (j = 0; j < editStreams.size(); j++) {
+ int numEditStreams = editStreams.size();
+ for (j = 0; j < numEditStreams; j++) {
if (editStreams.get(j) == eStream) {
break;
}
}
- if (j == editStreams.size()) {
+ if (j == numEditStreams) {
FSNamesystem.LOG.error("Unable to find sync log on which " +
" IO error occured. " +
"Fatal Error.");
@@ -841,7 +842,8 @@
synchronized void logEdit(byte op, Writable ... writables) {
assert this.getNumEditStreams() > 0 : "no editlog streams";
long start = FSNamesystem.now();
- for (int idx = 0; idx < editStreams.size(); idx++) {
+ int numEditStreams = editStreams.size();
+ for (int idx = 0; idx < numEditStreams; idx++) {
EditLogOutputStream eStream = editStreams.get(idx);
try {
eStream.write(op, writables);
@@ -874,11 +876,12 @@
long syncStart = 0;
// Fetch the transactionId of this thread.
- TransactionId id = myTransactionId.get();
- long mytxid = id.txid;
+ long mytxid = myTransactionId.get().txid;
+ final int numEditStreams;
synchronized (this) {
- assert this.getNumEditStreams() > 0 : "no editlog streams";
+ numEditStreams = editStreams.size();
+ assert numEditStreams > 0 : "no editlog streams";
printStatistics(false);
// if somebody is already syncing, then wait
@@ -901,15 +904,14 @@
isSyncRunning = true;
// swap buffers
- for (int idx = 0; idx < editStreams.size(); idx++) {
- EditLogOutputStream eStream = editStreams.get(idx);
- eStream.setReadyToFlush();
+ for (int idx = 0; idx < numEditStreams; idx++) {
+ editStreams.get(idx).setReadyToFlush();
}
}
// do the sync
long start = FSNamesystem.now();
- for (int idx = 0; idx < editStreams.size(); idx++) {
+ for (int idx = 0; idx < numEditStreams; idx++) {
EditLogOutputStream eStream = editStreams.get(idx);
try {
eStream.flush();
@@ -950,14 +952,17 @@
return;
}
lastPrintTime = now;
- StringBuffer buf = new StringBuffer();
-
- buf.append("Number of transactions: " + numTransactions +
- " Total time for transactions(ms): " +
- totalTimeTransactions);
- buf.append(" Number of syncs: " + editStreams.get(0).getNumSync());
+ StringBuilder buf = new StringBuilder();
+ buf.append("Number of transactions: ");
+ buf.append(numTransactions);
+ buf.append(" Total time for transactions(ms): ");
+ buf.append(totalTimeTransactions);
+ buf.append(" Number of syncs: ");
+ buf.append(editStreams.get(0).getNumSync());
buf.append(" SyncTimes(ms): ");
- for (int idx = 0; idx < editStreams.size(); idx++) {
+
+ int numEditStreams = editStreams.size();
+ for (int idx = 0; idx < numEditStreams; idx++) {
EditLogOutputStream eStream = editStreams.get(idx);
buf.append(eStream.getTotalSyncTime());
buf.append(" ");
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=712344&r1=712343&r2=712344&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Fri Nov 7 18:24:09 2008
@@ -1433,11 +1433,11 @@
* <code>inodes[inodes.length-1]</code> is the INode for the file.
*/
private Block allocateBlock(String src, INode[] inodes) throws IOException {
- Block b = null;
- do {
- b = new Block(FSNamesystem.randBlockId.nextLong(), 0,
- getGenerationStamp());
- } while (isValidBlock(b));
+ Block b = new Block(FSNamesystem.randBlockId.nextLong(), 0, 0);
+ while(isValidBlock(b)) {
+ b.setBlockId(FSNamesystem.randBlockId.nextLong());
+ }
+ b.setGenerationStamp(getGenerationStamp());
b = dir.addBlock(src, inodes, b);
NameNode.stateChangeLog.info("BLOCK* NameSystem.allocateBlock: "
+src+ ". "+b);
@@ -3879,7 +3879,7 @@
* @see SafeModeInfo
*/
private SafeModeInfo() {
- this.threshold = 1.5f; // this threshold can never be riched
+ this.threshold = 1.5f; // this threshold can never be reached
this.extension = 0;
this.safeReplication = Short.MAX_VALUE + 1; // more than maxReplication
this.blockTotal = -1;
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=712344&r1=712343&r2=712344&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
Fri Nov 7 18:24:09 2008
@@ -98,9 +98,7 @@
} else {
int size = this.blocks.length;
BlockInfo[] newlist = new BlockInfo[size + 1];
- for (int i = 0; i < size; i++) {
- newlist[i] = this.blocks[i];
- }
+ System.arraycopy(this.blocks, 0, newlist, 0, size);
newlist[size] = newblock;
this.blocks = newlist;
}
