Author: umamahesh
Date: Wed May 30 18:37:13 2012
New Revision: 1344397
URL: http://svn.apache.org/viewvc?rev=1344397&view=rev
Log:
Merge r:1344386 HDFS-3474. Cleanup Exception handling in BookKeeper journal
manager. Contributed by Ivan Kelly.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1344386
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1344386
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed May 30 18:37:13 2012
@@ -124,6 +124,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3398. Client will not retry when primaryDN is down once it's just got
pipeline.
(Amith D K via umamahesh)
+ HDFS-3474. Cleanup Exception handling in BookKeeper journal manager.
+ (Ivan Kelly via umamahesh)
+
Release 2.0.0-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
Wed May 30 18:37:13 2012
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.BKException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -104,8 +105,10 @@ class BookKeeperEditLogInputStream exten
public void close() throws IOException {
try {
lh.close();
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Exception closing ledger", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted closing ledger", e);
}
}
@@ -168,11 +171,8 @@ class BookKeeperEditLogInputStream exten
throws IOException {
this.lh = lh;
readEntries = firstBookKeeperEntry;
- try {
- maxEntry = lh.getLastAddConfirmed();
- } catch (Exception e) {
- throw new IOException("Error reading last entry id", e);
- }
+
+ maxEntry = lh.getLastAddConfirmed();
}
/**
@@ -193,8 +193,10 @@ class BookKeeperEditLogInputStream exten
assert !entries.hasMoreElements();
return e.getEntryInputStream();
}
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Error reading entries from bookkeeper", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted reading entries from bookkeeper",
e);
}
return null;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
Wed May 30 18:37:13 2012
@@ -188,8 +188,11 @@ public class BookKeeperJournalManager im
bkc = new BookKeeper(new ClientConfiguration(),
zkc);
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Error initializing zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while initializing bk journal
manager",
+ ie);
}
ci = new CurrentInprogress(zkc, currentInprogressNodePath);
@@ -211,6 +214,7 @@ public class BookKeeperJournalManager im
throw new IOException("We've already seen " + txId
+ ". A new stream cannot be created with it");
}
+
try {
String existingInprogressNode = ci.read();
if (null != existingInprogressNode
@@ -224,6 +228,15 @@ public class BookKeeperJournalManager im
currentLedger = bkc.createLedger(ensembleSize, quorumSize,
BookKeeper.DigestType.MAC,
digestpw.getBytes());
+ } catch (BKException bke) {
+ throw new IOException("Error creating ledger", bke);
+ } catch (KeeperException ke) {
+ throw new IOException("Error in zookeeper while creating ledger", ke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted creating ledger", ie);
+ }
+
+ try {
String znodePath = inprogressZNode(txId);
EditLogLedgerMetadata l = new EditLogLedgerMetadata(znodePath,
HdfsConstants.LAYOUT_VERSION, currentLedger.getId(), txId);
@@ -239,21 +252,27 @@ public class BookKeeperJournalManager im
maxTxId.store(txId);
ci.update(znodePath);
return new BookKeeperEditLogOutputStream(conf, currentLedger);
- } catch (Exception e) {
- if (currentLedger != null) {
- try {
- long id = currentLedger.getId();
- currentLedger.close();
- bkc.deleteLedger(id);
- } catch (Exception e2) {
- //log & ignore, an IOException will be thrown soon
- LOG.error("Error closing ledger", e2);
- }
- }
- throw new IOException("Error creating ledger", e);
+ } catch (KeeperException ke) {
+ cleanupLedger(currentLedger);
+ throw new IOException("Error storing ledger metadata", ke);
}
}
+ private void cleanupLedger(LedgerHandle lh) {
+ try {
+ long id = currentLedger.getId();
+ currentLedger.close();
+ bkc.deleteLedger(id);
+ } catch (BKException bke) {
+ //log & ignore, an IOException will be thrown soon
+ LOG.error("Error closing ledger", bke);
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted while closing ledger", ie);
+ }
+ }
+
+
+
/**
* Finalize a log segment. If the journal manager is currently
* writing to a ledger, ensure that this is the ledger of the log segment
@@ -347,8 +366,11 @@ public class BookKeeperJournalManager im
l);
s.skipTo(fromTxId);
return s;
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Could not open ledger for " + fromTxId, e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted opening ledger for "
+ + fromTxId, ie);
}
}
}
@@ -479,8 +501,10 @@ public class BookKeeperJournalManager im
try {
bkc.close();
zkc.close();
- } catch (Exception e) {
- throw new IOException("Couldn't close zookeeper client", e);
+ } catch (BKException bke) {
+ throw new IOException("Couldn't close bookkeeper client", bke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while closing journal manager", ie);
}
}
@@ -525,9 +549,12 @@ public class BookKeeperJournalManager im
op = in.readOp();
}
return endTxId;
- } catch (Exception e) {
+ } catch (BKException e) {
throw new IOException("Exception retreiving last tx id for ledger " + l,
e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while retreiving last tx id "
+ + "for ledger " + l, ie);
}
}
@@ -542,8 +569,10 @@ public class BookKeeperJournalManager im
for (String n : ledgerNames) {
ledgers.add(EditLogLedgerMetadata.read(zkc, ledgerPath + "/" + n));
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Exception reading ledger list from zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted getting list of ledgers from zk", ie);
}
Collections.sort(ledgers, EditLogLedgerMetadata.COMPARATOR);
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/CurrentInprogress.java
Wed May 30 18:37:13 2012
@@ -68,8 +68,10 @@ class CurrentInprogress {
}
}
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Exception accessing Zookeeper", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted accessing Zookeeper", ie);
}
}
@@ -158,4 +160,4 @@ class CurrentInprogress {
LOG.info("Cleared the data from CurrentInprogress");
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/EditLogLedgerMetadata.java
Wed May 30 18:37:13 2012
@@ -130,8 +130,10 @@ public class EditLogLedgerMetadata {
}
} catch(KeeperException.NoNodeException nne) {
throw nne;
- } catch(Exception e) {
- throw new IOException("Error reading from zookeeper", e);
+ } catch(KeeperException ke) {
+ throw new IOException("Error reading from zookeeper", ke);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted reading from zookeeper", ie);
}
}
@@ -151,9 +153,11 @@ public class EditLogLedgerMetadata {
Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException nee) {
throw nee;
- } catch (Exception e) {
- throw new IOException("Error creating ledger znode");
- }
+ } catch (KeeperException e) {
+ throw new IOException("Error creating ledger znode", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted creating ledger znode", ie);
+ }
}
boolean verify(ZooKeeper zkc, String path) {
@@ -164,9 +168,12 @@ public class EditLogLedgerMetadata {
+ " against " + other);
}
return other == this;
- } catch (Exception e) {
+ } catch (KeeperException e) {
LOG.error("Couldn't verify data in " + path, e);
return false;
+ } catch (IOException ie) {
+ LOG.error("Couldn't verify data in " + path, ie);
+ return false;
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java?rev=1344397&r1=1344396&r2=1344397&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/MaxTxId.java
Wed May 30 18:37:13 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.contrib.bkjour
import java.io.IOException;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
@@ -58,8 +59,10 @@ class MaxTxId {
zkc.create(path, txidStr.getBytes("UTF-8"),
Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Error writing max tx id", e);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted while writing max tx id", e);
}
}
}
@@ -74,8 +77,10 @@ class MaxTxId {
String txidString = new String(bytes, "UTF-8");
return Long.valueOf(txidString);
}
- } catch (Exception e) {
+ } catch (KeeperException e) {
throw new IOException("Error reading the max tx id from zk", e);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while reading thr max tx id", ie);
}
}
}
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1344386
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1344386
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1344386
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1344386
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1344386
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1344386
